fix: downloads, fix: logging space
Some checks failed
continuous-integration/drone/push Build is failing

This commit is contained in:
Evan Reichard 2024-01-22 17:43:47 -05:00
parent c9edcd8f5a
commit b49e0d19e9
3 changed files with 60 additions and 44 deletions

View File

@ -1166,13 +1166,13 @@ func (api *API) getDocumentsWordCount(documents []database.GetDocumentsWithStats
filePath := filepath.Join(api.Config.DataPath, "documents", *item.Filepath) filePath := filepath.Join(api.Config.DataPath, "documents", *item.Filepath)
wordCount, err := metadata.GetWordCount(filePath) wordCount, err := metadata.GetWordCount(filePath)
if err != nil { if err != nil {
log.Warn("[getDocumentsWordCount] Word Count Error - ", err) log.Warn("[getDocumentsWordCount] Word Count Error: ", err)
} else { } else {
if _, err := qtx.UpsertDocument(api.DB.Ctx, database.UpsertDocumentParams{ if _, err := qtx.UpsertDocument(api.DB.Ctx, database.UpsertDocumentParams{
ID: item.ID, ID: item.ID,
Words: &wordCount, Words: &wordCount,
}); err != nil { }); err != nil {
log.Error("[getDocumentsWordCount] UpsertDocument DB Error - ", err) log.Error("[getDocumentsWordCount] UpsertDocument DB Error: ", err)
return err return err
} }
} }

View File

@ -141,7 +141,6 @@ func (api *API) koSetProgress(c *gin.Context) {
return return
} }
start := time.Now()
// Upsert Device // Upsert Device
if _, err := api.DB.Queries.UpsertDevice(api.DB.Ctx, database.UpsertDeviceParams{ if _, err := api.DB.Queries.UpsertDevice(api.DB.Ctx, database.UpsertDeviceParams{
ID: rPosition.DeviceID, ID: rPosition.DeviceID,
@ -151,18 +150,14 @@ func (api *API) koSetProgress(c *gin.Context) {
}); err != nil { }); err != nil {
log.Error("[koSetProgress] UpsertDevice DB Error:", err) log.Error("[koSetProgress] UpsertDevice DB Error:", err)
} }
log.Debug("[koSetProgress] UpsertDevice Performance: ", time.Since(start))
start = time.Now()
// Upsert Document // Upsert Document
if _, err := api.DB.Queries.UpsertDocument(api.DB.Ctx, database.UpsertDocumentParams{ if _, err := api.DB.Queries.UpsertDocument(api.DB.Ctx, database.UpsertDocumentParams{
ID: rPosition.DocumentID, ID: rPosition.DocumentID,
}); err != nil { }); err != nil {
log.Error("[koSetProgress] UpsertDocument DB Error:", err) log.Error("[koSetProgress] UpsertDocument DB Error:", err)
} }
log.Debug("[koSetProgress] UpsertDocument Performance: ", time.Since(start))
start = time.Now()
// Create or Replace Progress // Create or Replace Progress
progress, err := api.DB.Queries.UpdateProgress(api.DB.Ctx, database.UpdateProgressParams{ progress, err := api.DB.Queries.UpdateProgress(api.DB.Ctx, database.UpdateProgressParams{
Percentage: rPosition.Percentage, Percentage: rPosition.Percentage,
@ -176,7 +171,6 @@ func (api *API) koSetProgress(c *gin.Context) {
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Request"}) c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Request"})
return return
} }
log.Debug("[koSetProgress] UpdateProgress Performance: ", time.Since(start))
c.JSON(http.StatusOK, gin.H{ c.JSON(http.StatusOK, gin.H{
"document": progress.DocumentID, "document": progress.DocumentID,
@ -197,12 +191,10 @@ func (api *API) koGetProgress(c *gin.Context) {
return return
} }
start := time.Now()
progress, err := api.DB.Queries.GetDocumentProgress(api.DB.Ctx, database.GetDocumentProgressParams{ progress, err := api.DB.Queries.GetDocumentProgress(api.DB.Ctx, database.GetDocumentProgressParams{
DocumentID: rDocID.DocumentID, DocumentID: rDocID.DocumentID,
UserID: auth.UserName, UserID: auth.UserName,
}) })
log.Debug("[koGetProgress] GetDocumentProgress Performance: ", time.Since(start))
if err == sql.ErrNoRows { if err == sql.ErrNoRows {
// Not Found // Not Found

View File

@ -1,6 +1,7 @@
package search package search
import ( import (
"crypto/tls"
"errors" "errors"
"fmt" "fmt"
"io" "io"
@ -14,6 +15,8 @@ import (
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
) )
const userAgent string = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:121.0) Gecko/20100101 Firefox/121.0"
type Cadence string type Cadence string
const ( const (
@ -112,10 +115,10 @@ func SaveBook(id string, source Source) (string, error) {
// Download File // Download File
log.Info("[SaveBook] Downloading Book: ", bookURL) log.Info("[SaveBook] Downloading Book: ", bookURL)
resp, err := http.Get(bookURL) resp, err := downloadBook(bookURL)
if err != nil { if err != nil {
os.Remove(tempFile.Name()) os.Remove(tempFile.Name())
log.Error("[SaveBook] Cover URL API Failure") log.Error("[SaveBook] Book URL API Failure: ", err)
return "", errors.New("API Failure") return "", errors.New("API Failure")
} }
defer resp.Body.Close() defer resp.Body.Close()
@ -125,7 +128,7 @@ func SaveBook(id string, source Source) (string, error) {
_, err = io.Copy(tempFile, resp.Body) _, err = io.Copy(tempFile, resp.Body)
if err != nil { if err != nil {
os.Remove(tempFile.Name()) os.Remove(tempFile.Name())
log.Error("[SaveBook] File Copy Error") log.Error("[SaveBook] File Copy Error: ", err)
return "", errors.New("File Failure") return "", errors.New("File Failure")
} }
@ -328,7 +331,10 @@ func parseAnnasArchiveDownloadURL(body io.ReadCloser) (string, error) {
return "", errors.New("Download URL not found") return "", errors.New("Download URL not found")
} }
return "http://libgen.li/" + downloadURL, nil // Possible Funky URL
downloadURL = strings.ReplaceAll(downloadURL, "\\", "/")
return downloadURL, nil
} }
func parseAnnasArchive(body io.ReadCloser) ([]SearchItem, error) { func parseAnnasArchive(body io.ReadCloser) ([]SearchItem, error) {
@ -379,3 +385,21 @@ func parseAnnasArchive(body io.ReadCloser) ([]SearchItem, error) {
// Return Results // Return Results
return allEntries, nil return allEntries, nil
} }
func downloadBook(bookURL string) (*http.Response, error) {
// Allow Insecure
client := &http.Client{Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}}
// Start Request
req, err := http.NewRequest("GET", bookURL, nil)
if err != nil {
return nil, err
}
// Set UserAgent
req.Header.Set("User-Agent", userAgent)
return client.Do(req)
}