fix: downloads, fix: logging space
All checks were successful
continuous-integration/drone/push Build is passing
All checks were successful
continuous-integration/drone/push Build is passing
This commit is contained in:
parent
c9edcd8f5a
commit
760b9ca0a0
@ -1166,13 +1166,13 @@ func (api *API) getDocumentsWordCount(documents []database.GetDocumentsWithStats
|
|||||||
filePath := filepath.Join(api.Config.DataPath, "documents", *item.Filepath)
|
filePath := filepath.Join(api.Config.DataPath, "documents", *item.Filepath)
|
||||||
wordCount, err := metadata.GetWordCount(filePath)
|
wordCount, err := metadata.GetWordCount(filePath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Warn("[getDocumentsWordCount] Word Count Error - ", err)
|
log.Warn("[getDocumentsWordCount] Word Count Error: ", err)
|
||||||
} else {
|
} else {
|
||||||
if _, err := qtx.UpsertDocument(api.DB.Ctx, database.UpsertDocumentParams{
|
if _, err := qtx.UpsertDocument(api.DB.Ctx, database.UpsertDocumentParams{
|
||||||
ID: item.ID,
|
ID: item.ID,
|
||||||
Words: &wordCount,
|
Words: &wordCount,
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
log.Error("[getDocumentsWordCount] UpsertDocument DB Error - ", err)
|
log.Error("[getDocumentsWordCount] UpsertDocument DB Error: ", err)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -141,7 +141,6 @@ func (api *API) koSetProgress(c *gin.Context) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
start := time.Now()
|
|
||||||
// Upsert Device
|
// Upsert Device
|
||||||
if _, err := api.DB.Queries.UpsertDevice(api.DB.Ctx, database.UpsertDeviceParams{
|
if _, err := api.DB.Queries.UpsertDevice(api.DB.Ctx, database.UpsertDeviceParams{
|
||||||
ID: rPosition.DeviceID,
|
ID: rPosition.DeviceID,
|
||||||
@ -151,18 +150,14 @@ func (api *API) koSetProgress(c *gin.Context) {
|
|||||||
}); err != nil {
|
}); err != nil {
|
||||||
log.Error("[koSetProgress] UpsertDevice DB Error:", err)
|
log.Error("[koSetProgress] UpsertDevice DB Error:", err)
|
||||||
}
|
}
|
||||||
log.Debug("[koSetProgress] UpsertDevice Performance: ", time.Since(start))
|
|
||||||
|
|
||||||
start = time.Now()
|
|
||||||
// Upsert Document
|
// Upsert Document
|
||||||
if _, err := api.DB.Queries.UpsertDocument(api.DB.Ctx, database.UpsertDocumentParams{
|
if _, err := api.DB.Queries.UpsertDocument(api.DB.Ctx, database.UpsertDocumentParams{
|
||||||
ID: rPosition.DocumentID,
|
ID: rPosition.DocumentID,
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
log.Error("[koSetProgress] UpsertDocument DB Error:", err)
|
log.Error("[koSetProgress] UpsertDocument DB Error:", err)
|
||||||
}
|
}
|
||||||
log.Debug("[koSetProgress] UpsertDocument Performance: ", time.Since(start))
|
|
||||||
|
|
||||||
start = time.Now()
|
|
||||||
// Create or Replace Progress
|
// Create or Replace Progress
|
||||||
progress, err := api.DB.Queries.UpdateProgress(api.DB.Ctx, database.UpdateProgressParams{
|
progress, err := api.DB.Queries.UpdateProgress(api.DB.Ctx, database.UpdateProgressParams{
|
||||||
Percentage: rPosition.Percentage,
|
Percentage: rPosition.Percentage,
|
||||||
@ -176,7 +171,6 @@ func (api *API) koSetProgress(c *gin.Context) {
|
|||||||
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Request"})
|
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Request"})
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
log.Debug("[koSetProgress] UpdateProgress Performance: ", time.Since(start))
|
|
||||||
|
|
||||||
c.JSON(http.StatusOK, gin.H{
|
c.JSON(http.StatusOK, gin.H{
|
||||||
"document": progress.DocumentID,
|
"document": progress.DocumentID,
|
||||||
@ -197,12 +191,10 @@ func (api *API) koGetProgress(c *gin.Context) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
start := time.Now()
|
|
||||||
progress, err := api.DB.Queries.GetDocumentProgress(api.DB.Ctx, database.GetDocumentProgressParams{
|
progress, err := api.DB.Queries.GetDocumentProgress(api.DB.Ctx, database.GetDocumentProgressParams{
|
||||||
DocumentID: rDocID.DocumentID,
|
DocumentID: rDocID.DocumentID,
|
||||||
UserID: auth.UserName,
|
UserID: auth.UserName,
|
||||||
})
|
})
|
||||||
log.Debug("[koGetProgress] GetDocumentProgress Performance: ", time.Since(start))
|
|
||||||
|
|
||||||
if err == sql.ErrNoRows {
|
if err == sql.ErrNoRows {
|
||||||
// Not Found
|
// Not Found
|
||||||
|
@ -1,6 +1,7 @@
|
|||||||
package search
|
package search
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"crypto/tls"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
@ -14,6 +15,8 @@ import (
|
|||||||
log "github.com/sirupsen/logrus"
|
log "github.com/sirupsen/logrus"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const userAgent string = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:121.0) Gecko/20100101 Firefox/121.0"
|
||||||
|
|
||||||
type Cadence string
|
type Cadence string
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@ -112,10 +115,10 @@ func SaveBook(id string, source Source) (string, error) {
|
|||||||
|
|
||||||
// Download File
|
// Download File
|
||||||
log.Info("[SaveBook] Downloading Book: ", bookURL)
|
log.Info("[SaveBook] Downloading Book: ", bookURL)
|
||||||
resp, err := http.Get(bookURL)
|
resp, err := downloadBook(bookURL)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
os.Remove(tempFile.Name())
|
os.Remove(tempFile.Name())
|
||||||
log.Error("[SaveBook] Cover URL API Failure")
|
log.Error("[SaveBook] Book URL API Failure: ", err)
|
||||||
return "", errors.New("API Failure")
|
return "", errors.New("API Failure")
|
||||||
}
|
}
|
||||||
defer resp.Body.Close()
|
defer resp.Body.Close()
|
||||||
@ -125,7 +128,7 @@ func SaveBook(id string, source Source) (string, error) {
|
|||||||
_, err = io.Copy(tempFile, resp.Body)
|
_, err = io.Copy(tempFile, resp.Body)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
os.Remove(tempFile.Name())
|
os.Remove(tempFile.Name())
|
||||||
log.Error("[SaveBook] File Copy Error")
|
log.Error("[SaveBook] File Copy Error: ", err)
|
||||||
return "", errors.New("File Failure")
|
return "", errors.New("File Failure")
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -328,7 +331,10 @@ func parseAnnasArchiveDownloadURL(body io.ReadCloser) (string, error) {
|
|||||||
return "", errors.New("Download URL not found")
|
return "", errors.New("Download URL not found")
|
||||||
}
|
}
|
||||||
|
|
||||||
return "http://libgen.li/" + downloadURL, nil
|
// Possible Funky URL
|
||||||
|
downloadURL = strings.ReplaceAll(downloadURL, "\\", "/")
|
||||||
|
|
||||||
|
return downloadURL, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func parseAnnasArchive(body io.ReadCloser) ([]SearchItem, error) {
|
func parseAnnasArchive(body io.ReadCloser) ([]SearchItem, error) {
|
||||||
@ -379,3 +385,21 @@ func parseAnnasArchive(body io.ReadCloser) ([]SearchItem, error) {
|
|||||||
// Return Results
|
// Return Results
|
||||||
return allEntries, nil
|
return allEntries, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func downloadBook(bookURL string) (*http.Response, error) {
|
||||||
|
// Allow Insecure
|
||||||
|
client := &http.Client{Transport: &http.Transport{
|
||||||
|
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
|
||||||
|
}}
|
||||||
|
|
||||||
|
// Start Request
|
||||||
|
req, err := http.NewRequest("GET", bookURL, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set UserAgent
|
||||||
|
req.Header.Set("User-Agent", userAgent)
|
||||||
|
|
||||||
|
return client.Do(req)
|
||||||
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user