From 75ed394f8d9f5eac60c5fff2da8e4b2c7352c8c9 Mon Sep 17 00:00:00 2001 From: Evan Reichard Date: Sat, 24 Feb 2024 20:45:26 -0500 Subject: [PATCH] tests(all): improve tests, refactor(api): saving books --- .drone.yml | 15 +- .gitignore | 1 + Makefile | 9 +- api/app-admin-routes.go | 90 +++++-- api/app-routes.go | 157 ++++-------- api/common.go | 2 +- api/ko-routes.go | 77 +++--- api/utils.go | 22 ++ api/utils_test.go | 35 ++- config/config.go | 1 + config/config_test.go | 46 ++-- database/manager_test.go | 20 +- go.mod | 4 + go.sum | 2 + metadata/_test_files/gbooks_id_response.json | 110 +++++++++ .../_test_files/gbooks_query_response.json | 105 ++++++++ metadata/epub.go | 1 + metadata/gbooks_test.go | 130 ++++++++++ metadata/integrations_test.go | 76 ------ metadata/metadata.go | 151 ++++++++++-- metadata/metadata_test.go | 54 +++-- opds/opds.go | 2 +- search/anna.go | 75 ++++++ search/goodreads.go | 42 ++++ search/libgen.go | 123 ++++++++++ search/search.go | 225 +----------------- utils/utils.go | 27 ++- utils/utils_test.go | 26 +- 28 files changed, 1033 insertions(+), 595 deletions(-) create mode 100644 metadata/_test_files/gbooks_id_response.json create mode 100644 metadata/_test_files/gbooks_query_response.json create mode 100644 metadata/gbooks_test.go delete mode 100644 metadata/integrations_test.go create mode 100644 search/anna.go create mode 100644 search/goodreads.go create mode 100644 search/libgen.go diff --git a/.drone.yml b/.drone.yml index 1759f35..42ec2c8 100644 --- a/.drone.yml +++ b/.drone.yml @@ -4,21 +4,10 @@ name: default steps: # Unit Tests - - name: unit test + - name: tests image: golang commands: - - make tests_unit - - # Integration Tests (Every Month) - - name: integration test - image: golang - commands: - - make tests_integration - when: - event: - - cron - cron: - - integration-test + - make tests # Fetch tags - name: fetch tags diff --git a/.gitignore b/.gitignore index 2913e15..93c8c8e 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ TODO.md data/ build/ .direnv/ +cover.html diff --git a/Makefile b/Makefile index 4d011f1..ee1a779 100644 --- a/Makefile +++ b/Makefile @@ -42,8 +42,7 @@ dev: build_tailwind clean: rm -rf ./build -tests_integration: - go test -v -tags=integration -coverpkg=./... ./metadata - -tests_unit: - SET_TEST=set_val go test -v -coverpkg=./... ./... +tests: + SET_TEST=set_val go test -coverpkg=./... ./... -coverprofile=./cover.out + go tool cover -html=./cover.out -o ./cover.html + rm ./cover.out diff --git a/api/app-admin-routes.go b/api/app-admin-routes.go index 8f70d6b..59211e2 100644 --- a/api/app-admin-routes.go +++ b/api/app-admin-routes.go @@ -19,6 +19,7 @@ import ( "github.com/gin-gonic/gin" "github.com/itchyny/gojq" log "github.com/sirupsen/logrus" + "reichard.io/antholume/metadata" ) type adminAction string @@ -63,7 +64,7 @@ func (api *API) appPerformAdminAction(c *gin.Context) { var rAdminAction requestAdminAction if err := c.ShouldBind(&rAdminAction); err != nil { log.Error("Invalid Form Bind: ", err) - appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values.") + appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values") return } @@ -75,6 +76,7 @@ func (api *API) appPerformAdminAction(c *gin.Context) { // 2. Select all / deselect? case adminCacheTables: go api.db.CacheTempTables() + // TODO - Message case adminRestore: api.processRestoreFile(rAdminAction, c) return @@ -83,7 +85,7 @@ func (api *API) appPerformAdminAction(c *gin.Context) { _, err := api.db.DB.ExecContext(api.db.Ctx, "VACUUM;") if err != nil { log.Error("Unable to vacuum DB: ", err) - appErrorPage(c, http.StatusInternalServerError, "Unable to vacuum database.") + appErrorPage(c, http.StatusInternalServerError, "Unable to vacuum database") return } @@ -126,7 +128,7 @@ func (api *API) appGetAdminLogs(c *gin.Context) { var rAdminLogs requestAdminLogs if err := c.ShouldBindQuery(&rAdminLogs); err != nil { log.Error("Invalid URI Bind") - appErrorPage(c, http.StatusNotFound, "Invalid URI parameters.") + appErrorPage(c, http.StatusNotFound, "Invalid URI parameters") return } rAdminLogs.Filter = strings.TrimSpace(rAdminLogs.Filter) @@ -136,14 +138,14 @@ func (api *API) appGetAdminLogs(c *gin.Context) { parsed, err := gojq.Parse(rAdminLogs.Filter) if err != nil { log.Error("Unable to parse JQ filter") - appErrorPage(c, http.StatusNotFound, "Unable to parse JQ filter.") + appErrorPage(c, http.StatusNotFound, "Unable to parse JQ filter") return } jqFilter, err = gojq.Compile(parsed) if err != nil { log.Error("Unable to compile JQ filter") - appErrorPage(c, http.StatusNotFound, "Unable to compile JQ filter.") + appErrorPage(c, http.StatusNotFound, "Unable to compile JQ filter") return } } @@ -152,7 +154,7 @@ func (api *API) appGetAdminLogs(c *gin.Context) { logPath := filepath.Join(api.cfg.ConfigPath, "logs/antholume.log") logFile, err := os.Open(logPath) if err != nil { - appErrorPage(c, http.StatusBadRequest, "Missing AnthoLume log file.") + appErrorPage(c, http.StatusBadRequest, "Missing AnthoLume log file") return } defer logFile.Close() @@ -229,7 +231,7 @@ func (api *API) appGetAdminImport(c *gin.Context) { var rImportFolder requestAdminImport if err := c.ShouldBindQuery(&rImportFolder); err != nil { log.Error("Invalid URI Bind") - appErrorPage(c, http.StatusNotFound, "Invalid directory.") + appErrorPage(c, http.StatusNotFound, "Invalid directory") return } @@ -244,7 +246,7 @@ func (api *API) appGetAdminImport(c *gin.Context) { dPath, err := filepath.Abs(api.cfg.DataPath) if err != nil { log.Error("Absolute filepath error: ", rImportFolder.Directory) - appErrorPage(c, http.StatusNotFound, "Unable to get data directory absolute path.") + appErrorPage(c, http.StatusNotFound, "Unable to get data directory absolute path") return } @@ -254,7 +256,7 @@ func (api *API) appGetAdminImport(c *gin.Context) { entries, err := os.ReadDir(rImportFolder.Directory) if err != nil { log.Error("Invalid directory: ", rImportFolder.Directory) - appErrorPage(c, http.StatusNotFound, "Invalid directory.") + appErrorPage(c, http.StatusNotFound, "Invalid directory") return } @@ -279,13 +281,46 @@ func (api *API) appPerformAdminImport(c *gin.Context) { var rAdminImport requestAdminImport if err := c.ShouldBind(&rAdminImport); err != nil { log.Error("Invalid URI Bind") - appErrorPage(c, http.StatusNotFound, "Invalid directory.") + appErrorPage(c, http.StatusNotFound, "Invalid directory") return } - // TODO + // TODO - Store results for approval? - fmt.Println(rAdminImport) + // Walk import directory & copy or import files + importDirectory := filepath.Clean(rAdminImport.Directory) + _ = filepath.WalkDir(importDirectory, func(currentPath string, f fs.DirEntry, err error) error { + if err != nil { + return err + } + if f.IsDir() { + return nil + } + + // Get metadata + fileMeta, err := metadata.GetMetadata(currentPath) + if err != nil { + fmt.Printf("metadata error: %v\n", err) + return nil + } + + // Only needed if copying + newName := deriveBaseFileName(fileMeta) + + // Open File on Disk + // file, err := os.Open(currentPath) + // if err != nil { + // return err + // } + // defer file.Close() + + // TODO - BasePath in DB + // TODO - Copy / Import + + fmt.Printf("New File Metadata: %s\n", newName) + + return nil + }) templateVars["CurrentPath"] = filepath.Clean(rAdminImport.Directory) @@ -297,14 +332,14 @@ func (api *API) processRestoreFile(rAdminAction requestAdminAction, c *gin.Conte uploadedFile, err := rAdminAction.RestoreFile.Open() if err != nil { log.Error("File Error: ", err) - appErrorPage(c, http.StatusInternalServerError, "Unable to open file.") + appErrorPage(c, http.StatusInternalServerError, "Unable to open file") return } fileMime, err := mimetype.DetectReader(uploadedFile) if err != nil { log.Error("MIME Error") - appErrorPage(c, http.StatusInternalServerError, "Unable to detect filetype.") + appErrorPage(c, http.StatusInternalServerError, "Unable to detect filetype") return } fileExtension := fileMime.Extension() @@ -312,7 +347,7 @@ func (api *API) processRestoreFile(rAdminAction requestAdminAction, c *gin.Conte // Validate Extension if !slices.Contains([]string{".zip"}, fileExtension) { log.Error("Invalid FileType: ", fileExtension) - appErrorPage(c, http.StatusBadRequest, "Invalid filetype.") + appErrorPage(c, http.StatusBadRequest, "Invalid filetype") return } @@ -320,7 +355,7 @@ func (api *API) processRestoreFile(rAdminAction requestAdminAction, c *gin.Conte tempFile, err := os.CreateTemp("", "restore") if err != nil { log.Warn("Temp File Create Error: ", err) - appErrorPage(c, http.StatusInternalServerError, "Unable to create temp file.") + appErrorPage(c, http.StatusInternalServerError, "Unable to create temp file") return } defer os.Remove(tempFile.Name()) @@ -330,7 +365,7 @@ func (api *API) processRestoreFile(rAdminAction requestAdminAction, c *gin.Conte err = c.SaveUploadedFile(rAdminAction.RestoreFile, tempFile.Name()) if err != nil { log.Error("File Error: ", err) - appErrorPage(c, http.StatusInternalServerError, "Unable to save file.") + appErrorPage(c, http.StatusInternalServerError, "Unable to save file") return } @@ -338,7 +373,7 @@ func (api *API) processRestoreFile(rAdminAction requestAdminAction, c *gin.Conte fileInfo, err := tempFile.Stat() if err != nil { log.Error("File Error: ", err) - appErrorPage(c, http.StatusInternalServerError, "Unable to read file.") + appErrorPage(c, http.StatusInternalServerError, "Unable to read file") return } @@ -346,7 +381,7 @@ func (api *API) processRestoreFile(rAdminAction requestAdminAction, c *gin.Conte zipReader, err := zip.NewReader(tempFile, fileInfo.Size()) if err != nil { log.Error("ZIP Error: ", err) - appErrorPage(c, http.StatusInternalServerError, "Unable to read zip.") + appErrorPage(c, http.StatusInternalServerError, "Unable to read zip") return } @@ -380,7 +415,7 @@ func (api *API) processRestoreFile(rAdminAction requestAdminAction, c *gin.Conte backupFile, err := os.Create(backupFilePath) if err != nil { log.Error("Unable to create backup file: ", err) - appErrorPage(c, http.StatusInternalServerError, "Unable to create backup file.") + appErrorPage(c, http.StatusInternalServerError, "Unable to create backup file") return } defer backupFile.Close() @@ -389,7 +424,7 @@ func (api *API) processRestoreFile(rAdminAction requestAdminAction, c *gin.Conte _, err = api.db.DB.ExecContext(api.db.Ctx, "VACUUM;") if err != nil { log.Error("Unable to vacuum DB: ", err) - appErrorPage(c, http.StatusInternalServerError, "Unable to vacuum database.") + appErrorPage(c, http.StatusInternalServerError, "Unable to vacuum database") return } @@ -398,7 +433,7 @@ func (api *API) processRestoreFile(rAdminAction requestAdminAction, c *gin.Conte err = api.createBackup(w, []string{"covers", "documents"}) if err != nil { log.Error("Unable to save backup file: ", err) - appErrorPage(c, http.StatusInternalServerError, "Unable to save backup file.") + appErrorPage(c, http.StatusInternalServerError, "Unable to save backup file") return } @@ -406,26 +441,26 @@ func (api *API) processRestoreFile(rAdminAction requestAdminAction, c *gin.Conte err = api.removeData() if err != nil { log.Error("Unable to delete data: ", err) - appErrorPage(c, http.StatusInternalServerError, "Unable to delete data.") + appErrorPage(c, http.StatusInternalServerError, "Unable to delete data") return } // Restore Data err = api.restoreData(zipReader) if err != nil { - appErrorPage(c, http.StatusInternalServerError, "Unable to restore data.") + appErrorPage(c, http.StatusInternalServerError, "Unable to restore data") log.Panic("Unable to restore data: ", err) } // Reinit DB if err := api.db.Reload(); err != nil { - appErrorPage(c, http.StatusInternalServerError, "Unable to reload DB.") + appErrorPage(c, http.StatusInternalServerError, "Unable to reload DB") log.Panicf("Unable to reload DB: %v", err) } // Rotate Auth Hashes if err := api.rotateAllAuthHashes(); err != nil { - appErrorPage(c, http.StatusInternalServerError, "Unable to rotate hashes.") + appErrorPage(c, http.StatusInternalServerError, "Unable to rotate hashes") log.Panicf("Unable to rotate auth hashes: %v", err) } @@ -433,6 +468,7 @@ func (api *API) processRestoreFile(rAdminAction requestAdminAction, c *gin.Conte c.Redirect(http.StatusFound, "/login") } +// Restore all data func (api *API) restoreData(zipReader *zip.Reader) error { // Ensure Directories api.cfg.EnsureDirectories() @@ -463,6 +499,7 @@ func (api *API) restoreData(zipReader *zip.Reader) error { return nil } +// Remove all data func (api *API) removeData() error { allPaths := []string{ "covers", @@ -485,6 +522,7 @@ func (api *API) removeData() error { return nil } +// Backup all data func (api *API) createBackup(w io.Writer, directories []string) error { ar := zip.NewWriter(w) diff --git a/api/app-routes.go b/api/app-routes.go index fd56818..32e2bd3 100644 --- a/api/app-routes.go +++ b/api/app-routes.go @@ -157,7 +157,7 @@ func (api *API) appGetDocument(c *gin.Context) { var rDocID requestDocumentID if err := c.ShouldBindUri(&rDocID); err != nil { log.Error("Invalid URI Bind") - appErrorPage(c, http.StatusNotFound, "Invalid document.") + appErrorPage(c, http.StatusNotFound, "Invalid document") return } @@ -361,7 +361,7 @@ func (api *API) appGetDocumentProgress(c *gin.Context) { var rDoc requestDocumentID if err := c.ShouldBindUri(&rDoc); err != nil { log.Error("Invalid URI Bind") - appErrorPage(c, http.StatusNotFound, "Invalid document.") + appErrorPage(c, http.StatusNotFound, "Invalid document") return } @@ -417,7 +417,7 @@ func (api *API) appUploadNewDocument(c *gin.Context) { var rDocUpload requestDocumentUpload if err := c.ShouldBind(&rDocUpload); err != nil { log.Error("Invalid Form Bind") - appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values.") + appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values") return } @@ -426,153 +426,92 @@ func (api *API) appUploadNewDocument(c *gin.Context) { return } - // Validate Type & Derive Extension on MIME - uploadedFile, err := rDocUpload.DocumentFile.Open() - if err != nil { - log.Error("File Error: ", err) - appErrorPage(c, http.StatusInternalServerError, "Unable to open file.") - return - } - - fileMime, err := mimetype.DetectReader(uploadedFile) - if err != nil { - log.Error("MIME Error") - appErrorPage(c, http.StatusInternalServerError, "Unable to detect filetype.") - return - } - fileExtension := fileMime.Extension() - - // Validate Extension - if !slices.Contains([]string{".epub"}, fileExtension) { - log.Error("Invalid FileType: ", fileExtension) - appErrorPage(c, http.StatusBadRequest, "Invalid filetype.") - return - } - // Create Temp File tempFile, err := os.CreateTemp("", "book") if err != nil { log.Warn("Temp File Create Error: ", err) - appErrorPage(c, http.StatusInternalServerError, "Unable to create temp file.") + appErrorPage(c, http.StatusInternalServerError, "Unable to create temp file") return } defer os.Remove(tempFile.Name()) defer tempFile.Close() - // Save Temp + // Save Temp File err = c.SaveUploadedFile(rDocUpload.DocumentFile, tempFile.Name()) if err != nil { log.Error("File Error: ", err) - appErrorPage(c, http.StatusInternalServerError, "Unable to save file.") + appErrorPage(c, http.StatusInternalServerError, "Unable to save file") return } // Get Metadata metadataInfo, err := metadata.GetMetadata(tempFile.Name()) if err != nil { - log.Warn("GetMetadata Error: ", err) - appErrorPage(c, http.StatusInternalServerError, "Unable to acquire file metadata.") + log.Errorf("unable to acquire metadata: %v", err) + appErrorPage(c, http.StatusInternalServerError, "Unable to acquire metadata") return } - // Calculate Partial MD5 ID - partialMD5, err := utils.CalculatePartialMD5(tempFile.Name()) - if err != nil { - log.Warn("Partial MD5 Error: ", err) - appErrorPage(c, http.StatusInternalServerError, "Unable to calculate partial MD5.") - return - } - - // Check Exists - _, err = api.db.Queries.GetDocument(api.db.Ctx, partialMD5) + // Check Already Exists + _, err = api.db.Queries.GetDocument(api.db.Ctx, *metadataInfo.PartialMD5) if err == nil { - c.Redirect(http.StatusFound, fmt.Sprintf("./documents/%s", partialMD5)) - return + log.Warnf("document already exists: %s", *metadataInfo.PartialMD5) + c.Redirect(http.StatusFound, fmt.Sprintf("./documents/%s", *metadataInfo.PartialMD5)) } - // Calculate Actual MD5 - fileHash, err := getFileMD5(tempFile.Name()) - if err != nil { - log.Error("MD5 Hash Failure: ", err) - appErrorPage(c, http.StatusInternalServerError, "Unable to calculate MD5.") - return - } - - // Get Word Count - wordCount, err := metadata.GetWordCount(tempFile.Name()) - if err != nil { - log.Error("Word Count Failure: ", err) - appErrorPage(c, http.StatusInternalServerError, "Unable to calculate word count.") - return - } - - // Derive Filename - var fileName string - if *metadataInfo.Author != "" { - fileName = fileName + *metadataInfo.Author - } else { - fileName = fileName + "Unknown" - } - - if *metadataInfo.Title != "" { - fileName = fileName + " - " + *metadataInfo.Title - } else { - fileName = fileName + " - Unknown" - } - - // Remove Slashes - fileName = strings.ReplaceAll(fileName, "/", "") - // Derive & Sanitize File Name - fileName = "." + filepath.Clean(fmt.Sprintf("/%s [%s]%s", fileName, partialMD5, fileExtension)) - - // Generate Storage Path & Open File + fileName := deriveBaseFileName(metadataInfo) safePath := filepath.Join(api.cfg.DataPath, "documents", fileName) + + // Open Destination File destFile, err := os.Create(safePath) if err != nil { - log.Error("Dest File Error: ", err) - appErrorPage(c, http.StatusInternalServerError, "Unable to save file.") + log.Errorf("unable to open destination file: %v", err) + appErrorPage(c, http.StatusInternalServerError, "Unable to open destination file") return } defer destFile.Close() // Copy File if _, err = io.Copy(destFile, tempFile); err != nil { - log.Error("Copy Temp File Error: ", err) - appErrorPage(c, http.StatusInternalServerError, "Unable to save file.") + log.Errorf("unable to save file: %v", err) + appErrorPage(c, http.StatusInternalServerError, "Unable to save file") return } // Upsert Document if _, err = api.db.Queries.UpsertDocument(api.db.Ctx, database.UpsertDocumentParams{ - ID: partialMD5, + ID: *metadataInfo.PartialMD5, Title: metadataInfo.Title, Author: metadataInfo.Author, Description: metadataInfo.Description, - Words: &wordCount, - Md5: fileHash, + Md5: metadataInfo.MD5, + Words: metadataInfo.WordCount, Filepath: &fileName, + + // TODO (BasePath): + // - Should be current config directory }); err != nil { - log.Error("UpsertDocument DB Error: ", err) + log.Errorf("UpsertDocument DB Error: %v", err) appErrorPage(c, http.StatusInternalServerError, fmt.Sprintf("UpsertDocument DB Error: %v", err)) return } - c.Redirect(http.StatusFound, fmt.Sprintf("./documents/%s", partialMD5)) + c.Redirect(http.StatusFound, fmt.Sprintf("./documents/%s", *metadataInfo.PartialMD5)) } func (api *API) appEditDocument(c *gin.Context) { var rDocID requestDocumentID if err := c.ShouldBindUri(&rDocID); err != nil { log.Error("Invalid URI Bind") - appErrorPage(c, http.StatusNotFound, "Invalid document.") + appErrorPage(c, http.StatusNotFound, "Invalid document") return } var rDocEdit requestDocumentEdit if err := c.ShouldBind(&rDocEdit); err != nil { log.Error("Invalid Form Bind") - appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values.") + appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values") return } @@ -586,7 +525,7 @@ func (api *API) appEditDocument(c *gin.Context) { rDocEdit.CoverGBID == nil && rDocEdit.CoverFile == nil { log.Error("Missing Form Values") - appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values.") + appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values") return } @@ -600,14 +539,14 @@ func (api *API) appEditDocument(c *gin.Context) { uploadedFile, err := rDocEdit.CoverFile.Open() if err != nil { log.Error("File Error") - appErrorPage(c, http.StatusInternalServerError, "Unable to open file.") + appErrorPage(c, http.StatusInternalServerError, "Unable to open file") return } fileMime, err := mimetype.DetectReader(uploadedFile) if err != nil { log.Error("MIME Error") - appErrorPage(c, http.StatusInternalServerError, "Unable to detect filetype.") + appErrorPage(c, http.StatusInternalServerError, "Unable to detect filetype") return } fileExtension := fileMime.Extension() @@ -615,7 +554,7 @@ func (api *API) appEditDocument(c *gin.Context) { // Validate Extension if !slices.Contains([]string{".jpg", ".png"}, fileExtension) { log.Error("Invalid FileType: ", fileExtension) - appErrorPage(c, http.StatusBadRequest, "Invalid filetype.") + appErrorPage(c, http.StatusBadRequest, "Invalid filetype") return } @@ -627,7 +566,7 @@ func (api *API) appEditDocument(c *gin.Context) { err = c.SaveUploadedFile(rDocEdit.CoverFile, safePath) if err != nil { log.Error("File Error: ", err) - appErrorPage(c, http.StatusInternalServerError, "Unable to save file.") + appErrorPage(c, http.StatusInternalServerError, "Unable to save file") return } @@ -663,7 +602,7 @@ func (api *API) appDeleteDocument(c *gin.Context) { var rDocID requestDocumentID if err := c.ShouldBindUri(&rDocID); err != nil { log.Error("Invalid URI Bind") - appErrorPage(c, http.StatusNotFound, "Invalid document.") + appErrorPage(c, http.StatusNotFound, "Invalid document") return } changed, err := api.db.Queries.DeleteDocument(api.db.Ctx, rDocID.DocumentID) @@ -674,7 +613,7 @@ func (api *API) appDeleteDocument(c *gin.Context) { } if changed == 0 { log.Error("DeleteDocument DB Error") - appErrorPage(c, http.StatusNotFound, "Invalid document.") + appErrorPage(c, http.StatusNotFound, "Invalid document") return } @@ -685,14 +624,14 @@ func (api *API) appIdentifyDocument(c *gin.Context) { var rDocID requestDocumentID if err := c.ShouldBindUri(&rDocID); err != nil { log.Error("Invalid URI Bind") - appErrorPage(c, http.StatusNotFound, "Invalid document.") + appErrorPage(c, http.StatusNotFound, "Invalid document") return } var rDocIdentify requestDocumentIdentify if err := c.ShouldBind(&rDocIdentify); err != nil { log.Error("Invalid Form Bind") - appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values.") + appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values") return } @@ -710,7 +649,7 @@ func (api *API) appIdentifyDocument(c *gin.Context) { // Validate Values if rDocIdentify.ISBN == nil && rDocIdentify.Title == nil && rDocIdentify.Author == nil { log.Error("Invalid Form") - appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values.") + appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values") return } @@ -718,7 +657,7 @@ func (api *API) appIdentifyDocument(c *gin.Context) { templateVars, auth := api.getBaseTemplateVars("document", c) // Get Metadata - metadataResults, err := metadata.SearchMetadata(metadata.GBOOK, metadata.MetadataInfo{ + metadataResults, err := metadata.SearchMetadata(metadata.SOURCE_GBOOK, metadata.MetadataInfo{ Title: rDocIdentify.Title, Author: rDocIdentify.Author, ISBN10: rDocIdentify.ISBN, @@ -767,7 +706,7 @@ func (api *API) appSaveNewDocument(c *gin.Context) { var rDocAdd requestDocumentAdd if err := c.ShouldBind(&rDocAdd); err != nil { log.Error("Invalid Form Bind") - appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values.") + appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values") return } @@ -845,7 +784,7 @@ func (api *API) appSaveNewDocument(c *gin.Context) { fileName = strings.ReplaceAll(fileName, "/", "") // Derive & Sanitize File Name - fileName = "." + filepath.Clean(fmt.Sprintf("/%s [%s]%s", fileName, partialMD5, fileExtension)) + fileName = "." + filepath.Clean(fmt.Sprintf("/%s [%s]%s", fileName, *partialMD5, fileExtension)) // Open Source File sourceFile, err := os.Open(tempFilePath) @@ -901,12 +840,12 @@ func (api *API) appSaveNewDocument(c *gin.Context) { // Upsert Document if _, err = api.db.Queries.UpsertDocument(api.db.Ctx, database.UpsertDocumentParams{ - ID: partialMD5, + ID: *partialMD5, Title: rDocAdd.Title, Author: rDocAdd.Author, Md5: fileHash, Filepath: &fileName, - Words: &wordCount, + Words: wordCount, }); err != nil { log.Error("UpsertDocument DB Error: ", err) sendDownloadMessage("Unable to save to database", gin.H{"Error": true}) @@ -917,7 +856,7 @@ func (api *API) appSaveNewDocument(c *gin.Context) { sendDownloadMessage("Download Success", gin.H{ "Progress": 100, "ButtonText": "Go to Book", - "ButtonHref": fmt.Sprintf("./documents/%s", partialMD5), + "ButtonHref": fmt.Sprintf("./documents/%s", *partialMD5), }) } @@ -925,14 +864,14 @@ func (api *API) appEditSettings(c *gin.Context) { var rUserSettings requestSettingsEdit if err := c.ShouldBind(&rUserSettings); err != nil { log.Error("Invalid Form Bind") - appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values.") + appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values") return } // Validate Something Exists if rUserSettings.Password == nil && rUserSettings.NewPassword == nil && rUserSettings.TimeOffset == nil { log.Error("Missing Form Values") - appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values.") + appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values") return } @@ -1023,7 +962,7 @@ func (api *API) getDocumentsWordCount(documents []database.GetDocumentsWithStats } else { if _, err := qtx.UpsertDocument(api.db.Ctx, database.UpsertDocumentParams{ ID: item.ID, - Words: &wordCount, + Words: wordCount, }); err != nil { log.Error("UpsertDocument DB Error: ", err) return err diff --git a/api/common.go b/api/common.go index f145dc7..6e1f7bc 100644 --- a/api/common.go +++ b/api/common.go @@ -95,7 +95,7 @@ func (api *API) createGetCoverHandler(errorFunc func(*gin.Context, int, string)) var coverFile string = "UNKNOWN" // Identify Documents & Save Covers - metadataResults, err := metadata.SearchMetadata(metadata.GBOOK, metadata.MetadataInfo{ + metadataResults, err := metadata.SearchMetadata(metadata.SOURCE_GBOOK, metadata.MetadataInfo{ Title: document.Title, Author: document.Author, }) diff --git a/api/ko-routes.go b/api/ko-routes.go index f07fcc0..ff5ccbb 100644 --- a/api/ko-routes.go +++ b/api/ko-routes.go @@ -10,13 +10,10 @@ import ( "net/http" "os" "path/filepath" - "strings" "time" - "github.com/gabriel-vasile/mimetype" "github.com/gin-gonic/gin" log "github.com/sirupsen/logrus" - "golang.org/x/exp/slices" "reichard.io/antholume/database" "reichard.io/antholume/metadata" ) @@ -456,21 +453,11 @@ func (api *API) koUploadExistingDocument(c *gin.Context) { return } + // Open Form File fileData, err := c.FormFile("file") if err != nil { log.Error("File Error:", err) - apiErrorPage(c, http.StatusBadRequest, "File Error") - return - } - - // Validate Type & Derive Extension on MIME - uploadedFile, err := fileData.Open() - fileMime, err := mimetype.DetectReader(uploadedFile) - fileExtension := fileMime.Extension() - - if !slices.Contains([]string{".epub", ".html"}, fileExtension) { - log.Error("Invalid FileType:", fileExtension) - apiErrorPage(c, http.StatusBadRequest, "Invalid Filetype") + apiErrorPage(c, http.StatusBadRequest, "File error") return } @@ -482,25 +469,29 @@ func (api *API) koUploadExistingDocument(c *gin.Context) { return } + // Open File + uploadedFile, err := fileData.Open() + if err != nil { + log.Error("Unable to open file") + apiErrorPage(c, http.StatusBadRequest, "Unable to open file") + return + } + + // Check Support + docType, err := metadata.GetDocumentTypeReader(uploadedFile) + if err != nil { + log.Error("Unsupported file") + apiErrorPage(c, http.StatusBadRequest, "Unsupported file") + return + } + // Derive Filename - var fileName string - if document.Author != nil { - fileName = fileName + *document.Author - } else { - fileName = fileName + "Unknown" - } - - if document.Title != nil { - fileName = fileName + " - " + *document.Title - } else { - fileName = fileName + " - Unknown" - } - - // Remove Slashes - fileName = strings.ReplaceAll(fileName, "/", "") - - // Derive & Sanitize File Name - fileName = "." + filepath.Clean(fmt.Sprintf("/%s [%s]%s", fileName, document.ID, fileExtension)) + fileName := deriveBaseFileName(&metadata.MetadataInfo{ + Type: *docType, + PartialMD5: &document.ID, + Title: document.Title, + Author: document.Author, + }) // Generate Storage Path safePath := filepath.Join(api.cfg.DataPath, "documents", fileName) @@ -516,28 +507,20 @@ func (api *API) koUploadExistingDocument(c *gin.Context) { } } - // Get MD5 Hash - fileHash, err := getFileMD5(safePath) + // Acquire Metadata + metadataInfo, err := metadata.GetMetadata(safePath) if err != nil { - log.Error("Hash Failure:", err) - apiErrorPage(c, http.StatusBadRequest, "File Error") - return - } - - // Get Word Count - wordCount, err := metadata.GetWordCount(safePath) - if err != nil { - log.Error("Word Count Failure:", err) - apiErrorPage(c, http.StatusBadRequest, "File Error") + log.Errorf("Unable to acquire metadata: %v", err) + apiErrorPage(c, http.StatusBadRequest, "Unable to acquire metadata") return } // Upsert Document if _, err = api.db.Queries.UpsertDocument(api.db.Ctx, database.UpsertDocumentParams{ ID: document.ID, - Md5: fileHash, + Md5: metadataInfo.MD5, + Words: metadataInfo.WordCount, Filepath: &fileName, - Words: &wordCount, }); err != nil { log.Error("UpsertDocument DB Error:", err) apiErrorPage(c, http.StatusBadRequest, "Document Error") diff --git a/api/utils.go b/api/utils.go index e9a6dd7..e918071 100644 --- a/api/utils.go +++ b/api/utils.go @@ -4,10 +4,13 @@ import ( "errors" "fmt" "math" + "path/filepath" "reflect" + "strings" "reichard.io/antholume/database" "reichard.io/antholume/graph" + "reichard.io/antholume/metadata" ) type UTCOffset struct { @@ -144,3 +147,22 @@ func fields(value interface{}) (map[string]interface{}, error) { } return m, nil } + +func deriveBaseFileName(metadataInfo *metadata.MetadataInfo) string { + // Derive New FileName + var newFileName string + if *metadataInfo.Author != "" { + newFileName = newFileName + *metadataInfo.Author + } else { + newFileName = newFileName + "Unknown" + } + if *metadataInfo.Title != "" { + newFileName = newFileName + " - " + *metadataInfo.Title + } else { + newFileName = newFileName + " - Unknown" + } + + // Remove Slashes + fileName := strings.ReplaceAll(newFileName, "/", "") + return "." + filepath.Clean(fmt.Sprintf("/%s [%s]%s", fileName, *metadataInfo.PartialMD5, metadataInfo.Type)) +} diff --git a/api/utils_test.go b/api/utils_test.go index f182faa..e620057 100644 --- a/api/utils_test.go +++ b/api/utils_test.go @@ -1,12 +1,35 @@ package api -import "testing" +import ( + "testing" + + "github.com/stretchr/testify/assert" +) func TestNiceSeconds(t *testing.T) { - want := "22d 7h 39m 31s" - nice := niceSeconds(1928371) + wantOne := "22d 7h 39m 31s" + wantNA := "N/A" - if nice != want { - t.Fatalf(`Expected: %v, Got: %v`, want, nice) - } + niceOne := niceSeconds(1928371) + niceNA := niceSeconds(0) + + assert.Equal(t, wantOne, niceOne, "should be nice seconds") + assert.Equal(t, wantNA, niceNA, "should be nice NA") +} + +func TestNiceNumbers(t *testing.T) { + wantMillions := "198M" + wantThousands := "19.8k" + wantThousandsTwo := "1.98k" + wantZero := "0" + + niceMillions := niceNumbers(198236461) + niceThousands := niceNumbers(19823) + niceThousandsTwo := niceNumbers(1984) + niceZero := niceNumbers(0) + + assert.Equal(t, wantMillions, niceMillions, "should be nice millions") + assert.Equal(t, wantThousands, niceThousands, "should be nice thousands") + assert.Equal(t, wantThousandsTwo, niceThousandsTwo, "should be nice thousands") + assert.Equal(t, wantZero, niceZero, "should be nice zero") } diff --git a/config/config.go b/config/config.go index a69b9d7..30a6355 100644 --- a/config/config.go +++ b/config/config.go @@ -118,6 +118,7 @@ func (c *Config) EnsureDirectories() { docDir := filepath.Join(c.DataPath, "documents") coversDir := filepath.Join(c.DataPath, "covers") backupDir := filepath.Join(c.DataPath, "backups") + os.Mkdir(docDir, 0755) os.Mkdir(coversDir, 0755) os.Mkdir(backupDir, 0755) diff --git a/config/config_test.go b/config/config_test.go index 353c9e6..072a043 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -1,35 +1,37 @@ package config -import "testing" +import ( + "runtime" + "testing" + + "github.com/stretchr/testify/assert" +) func TestLoadConfig(t *testing.T) { conf := Load() - want := "sqlite" - if conf.DBType != want { - t.Fatalf(`Load().DBType = %q, want match for %#q, nil`, conf.DBType, want) - } + assert.Equal(t, "sqlite", conf.DBType) } func TestGetEnvDefault(t *testing.T) { - want := "def_val" - envDefault := getEnv("DEFAULT_TEST", want) - if envDefault != want { - t.Fatalf(`getEnv("DEFAULT_TEST", "def_val") = %q, want match for %#q, nil`, envDefault, want) - } -} + desiredValue := "def_val" + envDefault := getEnv("DEFAULT_TEST", desiredValue) -func TestGetEnvSet(t *testing.T) { - envDefault := getEnv("SET_TEST", "not_this") - want := "set_val" - if envDefault != want { - t.Fatalf(`getEnv("SET_TEST", "not_this") = %q, want match for %#q, nil`, envDefault, want) - } + assert.Equal(t, desiredValue, envDefault) } func TestTrimLowerString(t *testing.T) { - want := "trimtest" - output := trimLowerString(" trimTest ") - if output != want { - t.Fatalf(`trimLowerString(" trimTest ") = %q, want match for %#q, nil`, output, want) - } + desiredValue := "trimtest" + outputValue := trimLowerString(" trimTest ") + + assert.Equal(t, desiredValue, outputValue) +} + +func TestPrettyCaller(t *testing.T) { + p, _, _, _ := runtime.Caller(0) + result := runtime.CallersFrames([]uintptr{p}) + f, _ := result.Next() + functionName, fileName := prettyCaller(&f) + + assert.Equal(t, "TestPrettyCaller", functionName, "should have current function name") + assert.Equal(t, "config/config_test.go@30", fileName, "should have current file path and line number") } diff --git a/database/manager_test.go b/database/manager_test.go index 31a02d3..5ab88d6 100644 --- a/database/manager_test.go +++ b/database/manager_test.go @@ -5,6 +5,7 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" "reichard.io/antholume/config" "reichard.io/antholume/utils" ) @@ -28,9 +29,7 @@ func TestNewMgr(t *testing.T) { } dbm := NewMgr(&cfg) - if dbm == nil { - t.Fatalf(`Expected: *DBManager, Got: nil`) - } + assert.NotNil(t, dbm, "should not be nil dbm") t.Run("Database", func(t *testing.T) { dt := databaseTest{t, dbm} @@ -46,9 +45,7 @@ func (dt *databaseTest) TestUser() { dt.Run("User", func(t *testing.T) { // Generate Auth Hash rawAuthHash, err := utils.GenerateToken(64) - if err != nil { - t.Fatalf(`Expected: %v, Got: %v, Error: %v`, nil, err, err) - } + assert.Nil(t, err, "should be nil err") authHash := fmt.Sprintf("%x", rawAuthHash) changed, err := dt.dbm.Queries.CreateUser(dt.dbm.Ctx, CreateUserParams{ @@ -57,14 +54,13 @@ func (dt *databaseTest) TestUser() { AuthHash: &authHash, }) - if err != nil || changed != 1 { - t.Fatalf(`Expected: %v, Got: %v, Error: %v`, 1, changed, err) - } + assert.Nil(t, err, "should be nil err") + assert.Equal(t, int64(1), changed) user, err := dt.dbm.Queries.GetUser(dt.dbm.Ctx, userID) - if err != nil || *user.Pass != userPass { - t.Fatalf(`Expected: %v, Got: %v, Error: %v`, userPass, *user.Pass, err) - } + + assert.Nil(t, err, "should be nil err") + assert.Equal(t, userPass, *user.Pass) }) } diff --git a/go.mod b/go.mod index 3091ad2..52e8e5e 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,7 @@ require ( github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect github.com/chenzhuoyu/iasm v0.9.1 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-playground/locales v0.14.1 // indirect @@ -39,6 +40,7 @@ require ( github.com/gorilla/securecookie v1.1.2 // indirect github.com/gorilla/sessions v1.2.2 // indirect github.com/itchyny/timefmt-go v0.1.5 // indirect + github.com/jarcoal/httpmock v1.3.1 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/klauspost/cpuid/v2 v2.2.6 // indirect @@ -47,9 +49,11 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sethvargo/go-retry v0.2.4 // indirect + github.com/stretchr/testify v1.8.4 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e // indirect diff --git a/go.sum b/go.sum index eaa290b..3825d23 100644 --- a/go.sum +++ b/go.sum @@ -127,6 +127,8 @@ github.com/jackc/pgx/v5 v5.5.1 h1:5I9etrGkLrN+2XPCsi6XLlV5DITbSL/xBZdmAxFcXPI= github.com/jackc/pgx/v5 v5.5.1/go.mod h1:Ig06C2Vu0t5qXC60W8sqIthScaEnFvojjj9dSljmHRA= github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jarcoal/httpmock v1.3.1 h1:iUx3whfZWVf3jT01hQTO/Eo5sAYtB2/rqaUuOtpInww= +github.com/jarcoal/httpmock v1.3.1/go.mod h1:3yb8rc4BI7TCBhFY8ng0gjuLKJNquuDNiPaZjnENuYg= github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4= github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak= github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4= diff --git a/metadata/_test_files/gbooks_id_response.json b/metadata/_test_files/gbooks_id_response.json new file mode 100644 index 0000000..15cd55d --- /dev/null +++ b/metadata/_test_files/gbooks_id_response.json @@ -0,0 +1,110 @@ +{ + "kind": "books#volume", + "id": "ZxwpakTv_MIC", + "etag": "mhqr3GsebaQ", + "selfLink": "https://www.googleapis.com/books/v1/volumes/ZxwpakTv_MIC", + "volumeInfo": { + "title": "Alice in Wonderland", + "authors": [ + "Lewis Carroll" + ], + "publisher": "The Floating Press", + "publishedDate": "2009-01-01", + "description": "Alice in Wonderland (also known as Alice's Adventures in Wonderland), from 1865, is the peculiar and imaginative tale of a girl who falls down a rabbit-hole into a bizarre world of eccentric and unusual creatures. Lewis Carroll's prominent example of the genre of \"literary nonsense\" has endured in popularity with its clever way of playing with logic and a narrative structure that has influence generations of fiction writing.", + "industryIdentifiers": [ + { + "type": "ISBN_10", + "identifier": "1877527815" + }, + { + "type": "ISBN_13", + "identifier": "9781877527814" + } + ], + "readingModes": { + "text": true, + "image": false + }, + "pageCount": 104, + "printedPageCount": 112, + "printType": "BOOK", + "categories": [ + "Fiction / Classics", + "Juvenile Fiction / General" + ], + "averageRating": 5, + "ratingsCount": 1, + "maturityRating": "NOT_MATURE", + "allowAnonLogging": true, + "contentVersion": "0.2.3.0.preview.2", + "panelizationSummary": { + "containsEpubBubbles": false, + "containsImageBubbles": false + }, + "imageLinks": { + "smallThumbnail": "http://books.google.com/books/content?id=ZxwpakTv_MIC&printsec=frontcover&img=1&zoom=5&edge=curl&imgtk=AFLRE71e5b-TeAKTiPSvXNUPeUi8rItzur2xSzwH8QU3qjKH0A2opmoq1o5I9RqJFt1BtcCCqILhnYRcB2aFLJmEvom11gx3Qn3PNN1iBLj2H5y2JHjM8wIwGT7iWFQmEn0Od7s6sOdk&source=gbs_api", + "thumbnail": "http://books.google.com/books/content?id=ZxwpakTv_MIC&printsec=frontcover&img=1&zoom=1&edge=curl&imgtk=AFLRE70QORt9J_DmKJgfyf9UEjQkdDMZ0qAu0GP315a1Q4CRS3snEjKnJJO2fYFdxjMwsSpmHoXDFPZbsy4gw-kMvF7lL8LtwxGbJGlfETHw_jbQBKBlKTrneK4XFvvV-EXNrZRgylxj&source=gbs_api", + "small": "http://books.google.com/books/content?id=ZxwpakTv_MIC&printsec=frontcover&img=1&zoom=2&edge=curl&imgtk=AFLRE70r1pAUt6VhuEEW8vXFhu8LvKln3yj0mdlaWPO4ZQuODLFQnH0fTebKMMX4ANR5i4PtC0oaI48XkwF-EdzlEM1WmUcR5383N4kRMXcta_i9nmb2y38dnh3hObwQW5VoAxbc9psn&source=gbs_api", + "medium": "http://books.google.com/books/content?id=ZxwpakTv_MIC&printsec=frontcover&img=1&zoom=3&edge=curl&imgtk=AFLRE7019EVuXvhzbhmtbz1QFh-ajB6kTKRHGhqijFf8big_GPRMMdpCdKlklFbkCfXvy8F64t5NKlThUHb3tFP-51bbDXkrVErFbCqKGzGnDSSm8cewqT8HiYDNHqn0hXYnuYvN4vYf&source=gbs_api", + "large": "http://books.google.com/books/content?id=ZxwpakTv_MIC&printsec=frontcover&img=1&zoom=4&edge=curl&imgtk=AFLRE72I15XZqp_8c8BAj4EskxkdC6nQz8F0Fs6VJhkykwIqfjzwuM34tUSQa3UnMGbx-UYjZjSLmCNFlePS8aR7yy-0UP9BRnYD-h5Qbesnnt_xdOb3u7Wdiobi6VbciNCBwUwbCyeH&source=gbs_api", + "extraLarge": "http://books.google.com/books/content?id=ZxwpakTv_MIC&printsec=frontcover&img=1&zoom=6&edge=curl&imgtk=AFLRE70rC6ktY6U0K_hqG1HxPl_9hMjpKb10p9DryVIwQgUjoJfWQOjpNA3EQ-5yk167yYDlO27gylqNAdJBYWu7ZHr3GuqkjTDpXjDvzBBppVyWaVNxKwhOz3gfJ-gzM6cC4kLHP26R&source=gbs_api" + }, + "language": "en", + "previewLink": "http://books.google.com/books?id=ZxwpakTv_MIC&hl=&source=gbs_api", + "infoLink": "https://play.google.com/store/books/details?id=ZxwpakTv_MIC&source=gbs_api", + "canonicalVolumeLink": "https://play.google.com/store/books/details?id=ZxwpakTv_MIC" + }, + "layerInfo": { + "layers": [ + { + "layerId": "geo", + "volumeAnnotationsVersion": "2" + } + ] + }, + "saleInfo": { + "country": "US", + "saleability": "FOR_SALE", + "isEbook": true, + "listPrice": { + "amount": 3.99, + "currencyCode": "USD" + }, + "retailPrice": { + "amount": 3.99, + "currencyCode": "USD" + }, + "buyLink": "https://play.google.com/store/books/details?id=ZxwpakTv_MIC&rdid=book-ZxwpakTv_MIC&rdot=1&source=gbs_api", + "offers": [ + { + "finskyOfferType": 1, + "listPrice": { + "amountInMicros": 3990000, + "currencyCode": "USD" + }, + "retailPrice": { + "amountInMicros": 3990000, + "currencyCode": "USD" + }, + "giftable": true + } + ] + }, + "accessInfo": { + "country": "US", + "viewability": "PARTIAL", + "embeddable": true, + "publicDomain": false, + "textToSpeechPermission": "ALLOWED", + "epub": { + "isAvailable": true, + "acsTokenLink": "http://books.google.com/books/download/Alice_in_Wonderland-sample-epub.acsm?id=ZxwpakTv_MIC&format=epub&output=acs4_fulfillment_token&dl_type=sample&source=gbs_api" + }, + "pdf": { + "isAvailable": false + }, + "webReaderLink": "http://play.google.com/books/reader?id=ZxwpakTv_MIC&hl=&source=gbs_api", + "accessViewStatus": "SAMPLE", + "quoteSharingAllowed": false + } +} diff --git a/metadata/_test_files/gbooks_query_response.json b/metadata/_test_files/gbooks_query_response.json new file mode 100644 index 0000000..465cb9d --- /dev/null +++ b/metadata/_test_files/gbooks_query_response.json @@ -0,0 +1,105 @@ +{ + "kind": "books#volumes", + "totalItems": 1, + "items": [ + { + "kind": "books#volume", + "id": "ZxwpakTv_MIC", + "etag": "F2eR9VV6VwQ", + "selfLink": "https://www.googleapis.com/books/v1/volumes/ZxwpakTv_MIC", + "volumeInfo": { + "title": "Alice in Wonderland", + "authors": [ + "Lewis Carroll" + ], + "publisher": "The Floating Press", + "publishedDate": "2009-01-01", + "description": "Alice in Wonderland (also known as Alice's Adventures in Wonderland), from 1865, is the peculiar and imaginative tale of a girl who falls down a rabbit-hole into a bizarre world of eccentric and unusual creatures. Lewis Carroll's prominent example of the genre of \"literary nonsense\" has endured in popularity with its clever way of playing with logic and a narrative structure that has influence generations of fiction writing.", + "industryIdentifiers": [ + { + "type": "ISBN_13", + "identifier": "9781877527814" + }, + { + "type": "ISBN_10", + "identifier": "1877527815" + } + ], + "readingModes": { + "text": true, + "image": false + }, + "pageCount": 104, + "printType": "BOOK", + "categories": [ + "Fiction" + ], + "averageRating": 5, + "ratingsCount": 1, + "maturityRating": "NOT_MATURE", + "allowAnonLogging": true, + "contentVersion": "0.2.3.0.preview.2", + "panelizationSummary": { + "containsEpubBubbles": false, + "containsImageBubbles": false + }, + "imageLinks": { + "smallThumbnail": "http://books.google.com/books/content?id=ZxwpakTv_MIC&printsec=frontcover&img=1&zoom=5&edge=curl&source=gbs_api", + "thumbnail": "http://books.google.com/books/content?id=ZxwpakTv_MIC&printsec=frontcover&img=1&zoom=1&edge=curl&source=gbs_api" + }, + "language": "en", + "previewLink": "http://books.google.com/books?id=ZxwpakTv_MIC&printsec=frontcover&dq=isbn:1877527815&hl=&cd=1&source=gbs_api", + "infoLink": "https://play.google.com/store/books/details?id=ZxwpakTv_MIC&source=gbs_api", + "canonicalVolumeLink": "https://play.google.com/store/books/details?id=ZxwpakTv_MIC" + }, + "saleInfo": { + "country": "US", + "saleability": "FOR_SALE", + "isEbook": true, + "listPrice": { + "amount": 3.99, + "currencyCode": "USD" + }, + "retailPrice": { + "amount": 3.99, + "currencyCode": "USD" + }, + "buyLink": "https://play.google.com/store/books/details?id=ZxwpakTv_MIC&rdid=book-ZxwpakTv_MIC&rdot=1&source=gbs_api", + "offers": [ + { + "finskyOfferType": 1, + "listPrice": { + "amountInMicros": 3990000, + "currencyCode": "USD" + }, + "retailPrice": { + "amountInMicros": 3990000, + "currencyCode": "USD" + }, + "giftable": true + } + ] + }, + "accessInfo": { + "country": "US", + "viewability": "PARTIAL", + "embeddable": true, + "publicDomain": false, + "textToSpeechPermission": "ALLOWED", + "epub": { + "isAvailable": true, + "acsTokenLink": "http://books.google.com/books/download/Alice_in_Wonderland-sample-epub.acsm?id=ZxwpakTv_MIC&format=epub&output=acs4_fulfillment_token&dl_type=sample&source=gbs_api" + }, + "pdf": { + "isAvailable": false + }, + "webReaderLink": "http://play.google.com/books/reader?id=ZxwpakTv_MIC&hl=&source=gbs_api", + "accessViewStatus": "SAMPLE", + "quoteSharingAllowed": false + }, + "searchInfo": { + "textSnippet": "Alice in Wonderland (also known as Alice's Adventures in Wonderland), from 1865, is the peculiar and imaginative tale of a girl who falls down a rabbit-hole into a bizarre world of eccentric and unusual creatures." + } + } + ] +} diff --git a/metadata/epub.go b/metadata/epub.go index b6e6a9d..19cda2e 100644 --- a/metadata/epub.go +++ b/metadata/epub.go @@ -16,6 +16,7 @@ func getEPUBMetadata(filepath string) (*MetadataInfo, error) { rf := rc.Rootfiles[0] parsedMetadata := &MetadataInfo{ + Type: TYPE_EPUB, Title: &rf.Title, Author: &rf.Creator, Description: &rf.Description, diff --git a/metadata/gbooks_test.go b/metadata/gbooks_test.go new file mode 100644 index 0000000..5b1fa0d --- /dev/null +++ b/metadata/gbooks_test.go @@ -0,0 +1,130 @@ +package metadata + +import ( + _ "embed" + "encoding/json" + "fmt" + "net/http" + "net/url" + "regexp" + "strings" + "testing" + + "github.com/jarcoal/httpmock" + "github.com/stretchr/testify/assert" +) + +// const GBOOKS_QUERY_URL string = "https://www.googleapis.com/books/v1/volumes?q=%s" +// const GBOOKS_GBID_INFO_URL string = "https://www.googleapis.com/books/v1/volumes/%s" +// const GBOOKS_GBID_COVER_URL string = "https://books.google.com/books/content/images/frontcover/%s?fife=w480-h690" + +//go:embed _test_files/gbooks_id_response.json +var idResp string + +//go:embed _test_files/gbooks_query_response.json +var queryResp string + +type details struct { + URLs []string +} + +// Hook API Helper +func hookAPI() *details { + // Start HTTPMock + httpmock.Activate() + + // Create details struct + d := &details{ + URLs: []string{}, + } + + // Create Hook + matchRE := regexp.MustCompile(`^https://www\.googleapis\.com/books/v1/volumes.*`) + httpmock.RegisterRegexpResponder("GET", matchRE, func(req *http.Request) (*http.Response, error) { + // Append URL + d.URLs = append(d.URLs, req.URL.String()) + + // Get Raw Response + var rawResp string + if req.URL.Query().Get("q") != "" { + rawResp = queryResp + } else { + rawResp = idResp + } + + // Convert to JSON Response + var responseData map[string]interface{} + json.Unmarshal([]byte(rawResp), &responseData) + + // Return Response + return httpmock.NewJsonResponse(200, responseData) + }) + + return d +} + +func TestGBooksGBIDMetadata(t *testing.T) { + hookDetails := hookAPI() + defer httpmock.DeactivateAndReset() + + GBID := "ZxwpakTv_MIC" + expectedURL := fmt.Sprintf(GBOOKS_GBID_INFO_URL, GBID) + metadataResp, err := getGBooksMetadata(MetadataInfo{ID: &GBID}) + + assert.Nil(t, err, "should not have error") + assert.Contains(t, hookDetails.URLs, expectedURL, "should have intercepted URL") + assert.Equal(t, 1, len(metadataResp), "should have single result") + + mResult := metadataResp[0] + validateResult(t, &mResult) +} + +func TestGBooksISBNQuery(t *testing.T) { + hookDetails := hookAPI() + defer httpmock.DeactivateAndReset() + + ISBN10 := "1877527815" + expectedURL := fmt.Sprintf(GBOOKS_QUERY_URL, "isbn:"+ISBN10) + metadataResp, err := getGBooksMetadata(MetadataInfo{ + ISBN10: &ISBN10, + }) + + assert.Nil(t, err, "should not have error") + assert.Contains(t, hookDetails.URLs, expectedURL, "should have intercepted URL") + assert.Equal(t, 1, len(metadataResp), "should have single result") + + mResult := metadataResp[0] + validateResult(t, &mResult) +} + +func TestGBooksTitleQuery(t *testing.T) { + hookDetails := hookAPI() + defer httpmock.DeactivateAndReset() + + title := "Alice in Wonderland 1877527815" + expectedURL := fmt.Sprintf(GBOOKS_QUERY_URL, url.QueryEscape(strings.TrimSpace(title))) + metadataResp, err := getGBooksMetadata(MetadataInfo{ + Title: &title, + }) + + assert.Nil(t, err, "should not have error") + assert.Contains(t, hookDetails.URLs, expectedURL, "should have intercepted URL") + assert.NotEqual(t, 0, len(metadataResp), "should not have no results") + + mResult := metadataResp[0] + validateResult(t, &mResult) +} + +func validateResult(t *testing.T, m *MetadataInfo) { + expectedTitle := "Alice in Wonderland" + expectedAuthor := "Lewis Carroll" + expectedDesc := "Alice in Wonderland (also known as Alice's Adventures in Wonderland), from 1865, is the peculiar and imaginative tale of a girl who falls down a rabbit-hole into a bizarre world of eccentric and unusual creatures. Lewis Carroll's prominent example of the genre of \"literary nonsense\" has endured in popularity with its clever way of playing with logic and a narrative structure that has influence generations of fiction writing." + expectedISBN10 := "1877527815" + expectedISBN13 := "9781877527814" + + assert.Equal(t, expectedTitle, *m.Title, "should have title") + assert.Equal(t, expectedAuthor, *m.Author, "should have author") + assert.Equal(t, expectedDesc, *m.Description, "should have description") + assert.Equal(t, expectedISBN10, *m.ISBN10, "should have ISBN10") + assert.Equal(t, expectedISBN13, *m.ISBN13, "should have ISBN10") +} diff --git a/metadata/integrations_test.go b/metadata/integrations_test.go deleted file mode 100644 index b120dd0..0000000 --- a/metadata/integrations_test.go +++ /dev/null @@ -1,76 +0,0 @@ -//go:build integration - -package metadata - -import ( - "testing" -) - -func TestGBooksGBIDMetadata(t *testing.T) { - GBID := "ZxwpakTv_MIC" - metadataResp, err := getGBooksMetadata(MetadataInfo{ - ID: &GBID, - }) - - if len(metadataResp) != 1 { - t.Fatalf(`Expected: %v, Got: %v, Error: %v`, 1, len(metadataResp), err) - } - - mResult := metadataResp[0] - validateResult(&mResult, t) -} - -func TestGBooksISBNQuery(t *testing.T) { - ISBN10 := "1877527815" - metadataResp, err := getGBooksMetadata(MetadataInfo{ - ISBN10: &ISBN10, - }) - - if len(metadataResp) != 1 { - t.Fatalf(`Expected: %v, Got: %v, Error: %v`, 1, len(metadataResp), err) - } - - mResult := metadataResp[0] - validateResult(&mResult, t) -} - -func TestGBooksTitleQuery(t *testing.T) { - title := "Alice in Wonderland 1877527815" - metadataResp, err := getGBooksMetadata(MetadataInfo{ - Title: &title, - }) - - if len(metadataResp) == 0 { - t.Fatalf(`Expected: %v, Got: %v, Error: %v`, "> 0", len(metadataResp), err) - } - - mResult := metadataResp[0] - validateResult(&mResult, t) -} - -func validateResult(m *MetadataInfo, t *testing.T) { - expect := "Lewis Carroll" - if *m.Author != expect { - t.Fatalf(`Expected: %v, Got: %v`, expect, *m.Author) - } - - expect = "Alice in Wonderland" - if *m.Title != expect { - t.Fatalf(`Expected: %v, Got: %v`, expect, *m.Title) - } - - expect = "Alice in Wonderland (also known as Alice's Adventures in Wonderland), from 1865, is the peculiar and imaginative tale of a girl who falls down a rabbit-hole into a bizarre world of eccentric and unusual creatures. Lewis Carroll's prominent example of the genre of \"literary nonsense\" has endured in popularity with its clever way of playing with logic and a narrative structure that has influence generations of fiction writing." - if *m.Description != expect { - t.Fatalf(`Expected: %v, Got: %v`, expect, *m.Description) - } - - expect = "1877527815" - if *m.ISBN10 != expect { - t.Fatalf(`Expected: %v, Got: %v`, expect, *m.ISBN10) - } - - expect = "9781877527814" - if *m.ISBN13 != expect { - t.Fatalf(`Expected: %v, Got: %v`, expect, *m.ISBN13) - } -} diff --git a/metadata/metadata.go b/metadata/metadata.go index 199e0b5..6806cea 100644 --- a/metadata/metadata.go +++ b/metadata/metadata.go @@ -3,27 +3,47 @@ package metadata import ( "errors" "fmt" + "io" "path/filepath" "github.com/gabriel-vasile/mimetype" + "reichard.io/antholume/utils" ) +type MetadataHandler func(string) (*MetadataInfo, error) + +type DocumentType string + +const ( + TYPE_EPUB DocumentType = ".epub" +) + +var extensionHandlerMap = map[DocumentType]MetadataHandler{ + TYPE_EPUB: getEPUBMetadata, +} + type Source int const ( - GBOOK Source = iota - OLIB + SOURCE_GBOOK Source = iota + SOURCE_OLIB ) type MetadataInfo struct { - ID *string + ID *string + MD5 *string + PartialMD5 *string + WordCount *int64 + Title *string Author *string Description *string ISBN10 *string ISBN13 *string + Type DocumentType } +// Downloads the Google Books cover file and saves it to the provided directory. func CacheCover(gbid string, coverDir string, documentID string, overwrite bool) (*string, error) { // Get Filepath coverFile := "." + filepath.Clean(fmt.Sprintf("/%s.jpg", documentID)) @@ -39,11 +59,12 @@ func CacheCover(gbid string, coverDir string, documentID string, overwrite bool) return &coverFile, nil } +// Searches source for metadata based on the provided information. func SearchMetadata(s Source, metadataSearch MetadataInfo) ([]MetadataInfo, error) { switch s { - case GBOOK: + case SOURCE_GBOOK: return getGBooksMetadata(metadataSearch) - case OLIB: + case SOURCE_OLIB: return nil, errors.New("Not implemented") default: return nil, errors.New("Not implemented") @@ -51,32 +72,112 @@ func SearchMetadata(s Source, metadataSearch MetadataInfo) ([]MetadataInfo, erro } } -func GetWordCount(filepath string) (int64, error) { - fileMime, err := mimetype.DetectFile(filepath) - if err != nil { - return 0, err - } - - if fileExtension := fileMime.Extension(); fileExtension == ".epub" { - totalWords, err := countEPUBWords(filepath) - if err != nil { - return 0, err - } - return totalWords, nil - } else { - return 0, errors.New("Invalid Extension") - } -} - -func GetMetadata(filepath string) (*MetadataInfo, error) { +// Returns the word count of the provided filepath. An error will be returned +// if the file is not supported. +func GetWordCount(filepath string) (*int64, error) { fileMime, err := mimetype.DetectFile(filepath) if err != nil { return nil, err } if fileExtension := fileMime.Extension(); fileExtension == ".epub" { - return getEPUBMetadata(filepath) + totalWords, err := countEPUBWords(filepath) + if err != nil { + return nil, err + } + return &totalWords, nil } else { - return nil, errors.New("Invalid Extension") + return nil, fmt.Errorf("Invalid extension") } } + +// Returns embedded metadata of the provided file. An error will be returned if +// the file is not supported. +func GetMetadata(filepath string) (*MetadataInfo, error) { + // Detect Extension Type + fileMime, err := mimetype.DetectFile(filepath) + if err != nil { + return nil, err + } + + // Get Extension Type Metadata Handler + fileExtension := fileMime.Extension() + handler, ok := extensionHandlerMap[DocumentType(fileExtension)] + if !ok { + return nil, fmt.Errorf("invalid extension %s", fileExtension) + } + + // Acquire Metadata + metadataInfo, err := handler(filepath) + + // Calculate MD5 & Partial MD5 + partialMD5, err := utils.CalculatePartialMD5(filepath) + if err != nil { + return nil, fmt.Errorf("unable to calculate partial MD5") + } + + // Calculate Actual MD5 + MD5, err := utils.CalculateMD5(filepath) + if err != nil { + return nil, fmt.Errorf("unable to calculate MD5") + } + + // Calculate Word Count + wordCount, err := GetWordCount(filepath) + if err != nil { + return nil, fmt.Errorf("unable to calculate word count") + } + + metadataInfo.WordCount = wordCount + metadataInfo.PartialMD5 = partialMD5 + metadataInfo.MD5 = MD5 + + return metadataInfo, nil +} + +// Returns the extension of the provided filepath (e.g. ".epub"). An error +// will be returned if the file is not supported. +func GetDocumentType(filepath string) (*DocumentType, error) { + // Detect Extension Type + fileMime, err := mimetype.DetectFile(filepath) + if err != nil { + return nil, err + } + + // Detect + fileExtension := fileMime.Extension() + docType, ok := ParseDocumentType(fileExtension) + if !ok { + return nil, fmt.Errorf("filetype not supported") + } + + return &docType, nil +} + +// Returns the extension of the provided file reader (e.g. ".epub"). An error +// will be returned if the file is not supported. +func GetDocumentTypeReader(r io.Reader) (*DocumentType, error) { + // Detect Extension Type + fileMime, err := mimetype.DetectReader(r) + if err != nil { + return nil, err + } + + // Detect + fileExtension := fileMime.Extension() + docType, ok := ParseDocumentType(fileExtension) + if !ok { + return nil, fmt.Errorf("filetype not supported") + } + + return &docType, nil +} + +// Given a filetype string, attempt to resolve a DocumentType +func ParseDocumentType(input string) (DocumentType, bool) { + validTypes := map[string]DocumentType{ + string(TYPE_EPUB): TYPE_EPUB, + } + found, ok := validTypes[input] + return found, ok +} diff --git a/metadata/metadata_test.go b/metadata/metadata_test.go index e4a47a4..b5b439c 100644 --- a/metadata/metadata_test.go +++ b/metadata/metadata_test.go @@ -1,36 +1,46 @@ package metadata import ( + "os" "testing" + + "github.com/stretchr/testify/assert" ) func TestGetWordCount(t *testing.T) { - var want int64 = 30080 - wordCount, err := countEPUBWords("../_test_files/alice.epub") + var desiredCount int64 = 30080 + actualCount, err := countEPUBWords("../_test_files/alice.epub") + + assert.Nil(t, err, "should have no error") + assert.Equal(t, desiredCount, actualCount, "should be correct word count") - if wordCount != want { - t.Fatalf(`Expected: %v, Got: %v, Error: %v`, want, wordCount, err) - } } func TestGetMetadata(t *testing.T) { - metadataInfo, err := getEPUBMetadata("../_test_files/alice.epub") - if err != nil { - t.Fatalf(`Expected: *MetadataInfo, Got: nil, Error: %v`, err) - } + desiredTitle := "Alice's Adventures in Wonderland / Illustrated by Arthur Rackham. With a Proem by Austin Dobson" + desiredAuthor := "Lewis Carroll" + desiredDescription := "" - want := "Alice's Adventures in Wonderland / Illustrated by Arthur Rackham. With a Proem by Austin Dobson" - if *metadataInfo.Title != want { - t.Fatalf(`Expected: %v, Got: %v, Error: %v`, want, *metadataInfo.Title, err) - } + metadataInfo, err := GetMetadata("../_test_files/alice.epub") - want = "Lewis Carroll" - if *metadataInfo.Author != want { - t.Fatalf(`Expected: %v, Got: %v, Error: %v`, want, *metadataInfo.Author, err) - } - - want = "" - if *metadataInfo.Description != want { - t.Fatalf(`Expected: %v, Got: %v, Error: %v`, want, *metadataInfo.Description, err) - } + assert.Nil(t, err, "should have no error") + assert.Equal(t, desiredTitle, *metadataInfo.Title, "should be correct title") + assert.Equal(t, desiredAuthor, *metadataInfo.Author, "should be correct author") + assert.Equal(t, desiredDescription, *metadataInfo.Description, "should be correct author") + assert.Equal(t, TYPE_EPUB, metadataInfo.Type, "should be correct type") +} + +func TestGetExtension(t *testing.T) { + docType, err := GetDocumentType("../_test_files/alice.epub") + + assert.Nil(t, err, "should have no error") + assert.Equal(t, TYPE_EPUB, *docType) +} + +func TestGetExtensionReader(t *testing.T) { + file, _ := os.Open("../_test_files/alice.epub") + docType, err := GetDocumentTypeReader(file) + + assert.Nil(t, err, "should have no error") + assert.Equal(t, TYPE_EPUB, *docType) } diff --git a/opds/opds.go b/opds/opds.go index f685376..7cf9525 100644 --- a/opds/opds.go +++ b/opds/opds.go @@ -8,8 +8,8 @@ import ( // Feed root element for acquisition or navigation feed type Feed struct { + ID string `xml:"id,omitempty"` XMLName xml.Name `xml:"feed"` - ID string `xml:"id,omitempty",` Title string `xml:"title,omitempty"` Updated time.Time `xml:"updated,omitempty"` Entries []Entry `xml:"entry,omitempty"` diff --git a/search/anna.go b/search/anna.go new file mode 100644 index 0000000..9d941a3 --- /dev/null +++ b/search/anna.go @@ -0,0 +1,75 @@ +package search + +import ( + "fmt" + "io" + "strings" + + "github.com/PuerkitoBio/goquery" +) + +func parseAnnasArchiveDownloadURL(body io.ReadCloser) (string, error) { + // Parse + defer body.Close() + doc, _ := goquery.NewDocumentFromReader(body) + + // Return Download URL + downloadURL, exists := doc.Find("body > table > tbody > tr > td > a").Attr("href") + if exists == false { + return "", fmt.Errorf("Download URL not found") + } + + // Possible Funky URL + downloadURL = strings.ReplaceAll(downloadURL, "\\", "/") + + return downloadURL, nil +} + +func parseAnnasArchive(body io.ReadCloser) ([]SearchItem, error) { + // Parse + defer body.Close() + doc, err := goquery.NewDocumentFromReader(body) + if err != nil { + return nil, err + } + + // Normalize Results + var allEntries []SearchItem + doc.Find("form > div.w-full > div.w-full > div > div.justify-center").Each(func(ix int, rawBook *goquery.Selection) { + // Parse Details + details := rawBook.Find("div:nth-child(2) > div:nth-child(1)").Text() + detailsSplit := strings.Split(details, ", ") + + // Invalid Details + if len(detailsSplit) < 3 { + return + } + + language := detailsSplit[0] + fileType := detailsSplit[1] + fileSize := detailsSplit[2] + + // Get Title & Author + title := rawBook.Find("h3").Text() + author := rawBook.Find("div:nth-child(2) > div:nth-child(4)").Text() + + // Parse MD5 + itemHref, _ := rawBook.Find("a").Attr("href") + hrefArray := strings.Split(itemHref, "/") + id := hrefArray[len(hrefArray)-1] + + item := SearchItem{ + ID: id, + Title: title, + Author: author, + Language: language, + FileType: fileType, + FileSize: fileSize, + } + + allEntries = append(allEntries, item) + }) + + // Return Results + return allEntries, nil +} diff --git a/search/goodreads.go b/search/goodreads.go new file mode 100644 index 0000000..71c3e33 --- /dev/null +++ b/search/goodreads.go @@ -0,0 +1,42 @@ +package search + +import ( + "io" + + "github.com/PuerkitoBio/goquery" +) + +func GoodReadsMostRead(c Cadence) ([]SearchItem, error) { + body, err := getPage("https://www.goodreads.com/book/most_read?category=all&country=US&duration=" + string(c)) + if err != nil { + return nil, err + } + return parseGoodReads(body) +} + +func parseGoodReads(body io.ReadCloser) ([]SearchItem, error) { + // Parse + defer body.Close() + doc, err := goquery.NewDocumentFromReader(body) + if err != nil { + return nil, err + } + + // Normalize Results + var allEntries []SearchItem + + doc.Find("[itemtype=\"http://schema.org/Book\"]").Each(func(ix int, rawBook *goquery.Selection) { + title := rawBook.Find(".bookTitle span").Text() + author := rawBook.Find(".authorName span").Text() + + item := SearchItem{ + Title: title, + Author: author, + } + + allEntries = append(allEntries, item) + }) + + // Return Results + return allEntries, nil +} diff --git a/search/libgen.go b/search/libgen.go new file mode 100644 index 0000000..7a9243d --- /dev/null +++ b/search/libgen.go @@ -0,0 +1,123 @@ +package search + +import ( + "fmt" + "io" + "strings" + "time" + + "github.com/PuerkitoBio/goquery" +) + +func parseLibGenFiction(body io.ReadCloser) ([]SearchItem, error) { + // Parse + defer body.Close() + doc, err := goquery.NewDocumentFromReader(body) + if err != nil { + return nil, err + } + + // Normalize Results + var allEntries []SearchItem + doc.Find("table.catalog tbody > tr").Each(func(ix int, rawBook *goquery.Selection) { + + // Parse File Details + fileItem := rawBook.Find("td:nth-child(5)") + fileDesc := fileItem.Text() + fileDescSplit := strings.Split(fileDesc, "/") + fileType := strings.ToLower(strings.TrimSpace(fileDescSplit[0])) + fileSize := strings.TrimSpace(fileDescSplit[1]) + + // Parse Upload Date + uploadedRaw, _ := fileItem.Attr("title") + uploadedDateRaw := strings.Split(uploadedRaw, "Uploaded at ")[1] + uploadDate, _ := time.Parse("2006-01-02 15:04:05", uploadedDateRaw) + + // Parse MD5 + editHref, _ := rawBook.Find("td:nth-child(7) a").Attr("href") + hrefArray := strings.Split(editHref, "/") + id := hrefArray[len(hrefArray)-1] + + // Parse Other Details + title := rawBook.Find("td:nth-child(3) p a").Text() + author := rawBook.Find(".catalog_authors li a").Text() + language := rawBook.Find("td:nth-child(4)").Text() + series := rawBook.Find("td:nth-child(2)").Text() + + item := SearchItem{ + ID: id, + Title: title, + Author: author, + Series: series, + Language: language, + FileType: fileType, + FileSize: fileSize, + UploadDate: uploadDate.Format(time.RFC3339), + } + + allEntries = append(allEntries, item) + }) + + // Return Results + return allEntries, nil +} + +func parseLibGenNonFiction(body io.ReadCloser) ([]SearchItem, error) { + // Parse + defer body.Close() + doc, err := goquery.NewDocumentFromReader(body) + if err != nil { + return nil, err + } + + // Normalize Results + var allEntries []SearchItem + doc.Find("table.c tbody > tr:nth-child(n + 2)").Each(func(ix int, rawBook *goquery.Selection) { + + // Parse Type & Size + fileSize := strings.ToLower(strings.TrimSpace(rawBook.Find("td:nth-child(8)").Text())) + fileType := strings.ToLower(strings.TrimSpace(rawBook.Find("td:nth-child(9)").Text())) + + // Parse MD5 + titleRaw := rawBook.Find("td:nth-child(3) [id]") + editHref, _ := titleRaw.Attr("href") + hrefArray := strings.Split(editHref, "?md5=") + id := hrefArray[1] + + // Parse Other Details + title := titleRaw.Text() + author := rawBook.Find("td:nth-child(2)").Text() + language := rawBook.Find("td:nth-child(7)").Text() + series := rawBook.Find("td:nth-child(3) [href*='column=series']").Text() + + item := SearchItem{ + ID: id, + Title: title, + Author: author, + Series: series, + Language: language, + FileType: fileType, + FileSize: fileSize, + } + + allEntries = append(allEntries, item) + }) + + // Return Results + return allEntries, nil +} + +func parseLibGenDownloadURL(body io.ReadCloser) (string, error) { + // Parse + defer body.Close() + doc, _ := goquery.NewDocumentFromReader(body) + + // Return Download URL + // downloadURL, _ := doc.Find("#download [href*=cloudflare]").Attr("href") + downloadURL, exists := doc.Find("#download h2 a").Attr("href") + if exists == false { + return "", fmt.Errorf("Download URL not found") + } + + return downloadURL, nil +} diff --git a/search/search.go b/search/search.go index 98f1e5c..02e4015 100644 --- a/search/search.go +++ b/search/search.go @@ -2,16 +2,13 @@ package search import ( "crypto/tls" - "errors" "fmt" "io" "net/http" "net/url" "os" - "strings" "time" - "github.com/PuerkitoBio/goquery" log "github.com/sirupsen/logrus" ) @@ -102,14 +99,14 @@ func SaveBook(id string, source Source) (string, error) { bookURL, err := def.parseDownloadFunc(body) if err != nil { log.Error("Parse Download URL Error: ", err) - return "", errors.New("Download Failure") + return "", fmt.Errorf("Download Failure") } // Create File tempFile, err := os.CreateTemp("", "book") if err != nil { log.Error("File Create Error: ", err) - return "", errors.New("File Failure") + return "", fmt.Errorf("File Failure") } defer tempFile.Close() @@ -119,7 +116,7 @@ func SaveBook(id string, source Source) (string, error) { if err != nil { os.Remove(tempFile.Name()) log.Error("Book URL API Failure: ", err) - return "", errors.New("API Failure") + return "", fmt.Errorf("API Failure") } defer resp.Body.Close() @@ -129,20 +126,12 @@ func SaveBook(id string, source Source) (string, error) { if err != nil { os.Remove(tempFile.Name()) log.Error("File Copy Error: ", err) - return "", errors.New("File Failure") + return "", fmt.Errorf("File Failure") } return tempFile.Name(), nil } -func GoodReadsMostRead(c Cadence) ([]SearchItem, error) { - body, err := getPage("https://www.goodreads.com/book/most_read?category=all&country=US&duration=" + string(c)) - if err != nil { - return nil, err - } - return parseGoodReads(body) -} - func GetBookURL(id string, bookType BookType) (string, error) { // Derive Info URL var infoURL string @@ -180,212 +169,6 @@ func getPage(page string) (io.ReadCloser, error) { return resp.Body, err } -func parseLibGenFiction(body io.ReadCloser) ([]SearchItem, error) { - // Parse - defer body.Close() - doc, err := goquery.NewDocumentFromReader(body) - if err != nil { - return nil, err - } - - // Normalize Results - var allEntries []SearchItem - doc.Find("table.catalog tbody > tr").Each(func(ix int, rawBook *goquery.Selection) { - - // Parse File Details - fileItem := rawBook.Find("td:nth-child(5)") - fileDesc := fileItem.Text() - fileDescSplit := strings.Split(fileDesc, "/") - fileType := strings.ToLower(strings.TrimSpace(fileDescSplit[0])) - fileSize := strings.TrimSpace(fileDescSplit[1]) - - // Parse Upload Date - uploadedRaw, _ := fileItem.Attr("title") - uploadedDateRaw := strings.Split(uploadedRaw, "Uploaded at ")[1] - uploadDate, _ := time.Parse("2006-01-02 15:04:05", uploadedDateRaw) - - // Parse MD5 - editHref, _ := rawBook.Find("td:nth-child(7) a").Attr("href") - hrefArray := strings.Split(editHref, "/") - id := hrefArray[len(hrefArray)-1] - - // Parse Other Details - title := rawBook.Find("td:nth-child(3) p a").Text() - author := rawBook.Find(".catalog_authors li a").Text() - language := rawBook.Find("td:nth-child(4)").Text() - series := rawBook.Find("td:nth-child(2)").Text() - - item := SearchItem{ - ID: id, - Title: title, - Author: author, - Series: series, - Language: language, - FileType: fileType, - FileSize: fileSize, - UploadDate: uploadDate.Format(time.RFC3339), - } - - allEntries = append(allEntries, item) - }) - - // Return Results - return allEntries, nil -} - -func parseLibGenNonFiction(body io.ReadCloser) ([]SearchItem, error) { - // Parse - defer body.Close() - doc, err := goquery.NewDocumentFromReader(body) - if err != nil { - return nil, err - } - - // Normalize Results - var allEntries []SearchItem - doc.Find("table.c tbody > tr:nth-child(n + 2)").Each(func(ix int, rawBook *goquery.Selection) { - - // Parse Type & Size - fileSize := strings.ToLower(strings.TrimSpace(rawBook.Find("td:nth-child(8)").Text())) - fileType := strings.ToLower(strings.TrimSpace(rawBook.Find("td:nth-child(9)").Text())) - - // Parse MD5 - titleRaw := rawBook.Find("td:nth-child(3) [id]") - editHref, _ := titleRaw.Attr("href") - hrefArray := strings.Split(editHref, "?md5=") - id := hrefArray[1] - - // Parse Other Details - title := titleRaw.Text() - author := rawBook.Find("td:nth-child(2)").Text() - language := rawBook.Find("td:nth-child(7)").Text() - series := rawBook.Find("td:nth-child(3) [href*='column=series']").Text() - - item := SearchItem{ - ID: id, - Title: title, - Author: author, - Series: series, - Language: language, - FileType: fileType, - FileSize: fileSize, - } - - allEntries = append(allEntries, item) - }) - - // Return Results - return allEntries, nil -} - -func parseLibGenDownloadURL(body io.ReadCloser) (string, error) { - // Parse - defer body.Close() - doc, _ := goquery.NewDocumentFromReader(body) - - // Return Download URL - // downloadURL, _ := doc.Find("#download [href*=cloudflare]").Attr("href") - downloadURL, exists := doc.Find("#download h2 a").Attr("href") - if exists == false { - return "", errors.New("Download URL not found") - } - - return downloadURL, nil -} - -func parseGoodReads(body io.ReadCloser) ([]SearchItem, error) { - // Parse - defer body.Close() - doc, err := goquery.NewDocumentFromReader(body) - if err != nil { - return nil, err - } - - // Normalize Results - var allEntries []SearchItem - - doc.Find("[itemtype=\"http://schema.org/Book\"]").Each(func(ix int, rawBook *goquery.Selection) { - title := rawBook.Find(".bookTitle span").Text() - author := rawBook.Find(".authorName span").Text() - - item := SearchItem{ - Title: title, - Author: author, - } - - allEntries = append(allEntries, item) - }) - - // Return Results - return allEntries, nil -} - -func parseAnnasArchiveDownloadURL(body io.ReadCloser) (string, error) { - // Parse - defer body.Close() - doc, _ := goquery.NewDocumentFromReader(body) - - // Return Download URL - downloadURL, exists := doc.Find("body > table > tbody > tr > td > a").Attr("href") - if exists == false { - return "", errors.New("Download URL not found") - } - - // Possible Funky URL - downloadURL = strings.ReplaceAll(downloadURL, "\\", "/") - - return downloadURL, nil -} - -func parseAnnasArchive(body io.ReadCloser) ([]SearchItem, error) { - // Parse - defer body.Close() - doc, err := goquery.NewDocumentFromReader(body) - if err != nil { - return nil, err - } - - // Normalize Results - var allEntries []SearchItem - doc.Find("form > div.w-full > div.w-full > div > div.justify-center").Each(func(ix int, rawBook *goquery.Selection) { - // Parse Details - details := rawBook.Find("div:nth-child(2) > div:nth-child(1)").Text() - detailsSplit := strings.Split(details, ", ") - - // Invalid Details - if len(detailsSplit) < 3 { - return - } - - language := detailsSplit[0] - fileType := detailsSplit[1] - fileSize := detailsSplit[2] - - // Get Title & Author - title := rawBook.Find("h3").Text() - author := rawBook.Find("div:nth-child(2) > div:nth-child(4)").Text() - - // Parse MD5 - itemHref, _ := rawBook.Find("a").Attr("href") - hrefArray := strings.Split(itemHref, "/") - id := hrefArray[len(hrefArray)-1] - - item := SearchItem{ - ID: id, - Title: title, - Author: author, - Language: language, - FileType: fileType, - FileSize: fileSize, - } - - allEntries = append(allEntries, item) - }) - - // Return Results - return allEntries, nil -} - func downloadBook(bookURL string) (*http.Response, error) { // Allow Insecure client := &http.Client{Transport: &http.Transport{ diff --git a/utils/utils.go b/utils/utils.go index 07fa056..0f2265d 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -10,10 +10,10 @@ import ( ) // Reimplemented KOReader Partial MD5 Calculation -func CalculatePartialMD5(filePath string) (string, error) { +func CalculatePartialMD5(filePath string) (*string, error) { file, err := os.Open(filePath) if err != nil { - return "", err + return nil, err } defer file.Close() @@ -41,7 +41,8 @@ func CalculatePartialMD5(filePath string) (string, error) { } allBytes := buf.Bytes() - return fmt.Sprintf("%x", md5.Sum(allBytes)), nil + fileHash := fmt.Sprintf("%x", md5.Sum(allBytes)) + return &fileHash, nil } // Creates a token of n size @@ -53,3 +54,23 @@ func GenerateToken(n int) ([]byte, error) { } return b, nil } + +// Calculate MD5 of a file +func CalculateMD5(filePath string) (*string, error) { + file, err := os.Open(filePath) + if err != nil { + return nil, err + } + + defer file.Close() + + hash := md5.New() + _, err = io.Copy(hash, file) + if err != nil { + return nil, err + } + + fileHash := fmt.Sprintf("%x", hash.Sum(nil)) + + return &fileHash, nil +} diff --git a/utils/utils_test.go b/utils/utils_test.go index a3c73d5..97d0a54 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -1,12 +1,26 @@ package utils -import "testing" +import ( + "github.com/stretchr/testify/assert" + "testing" +) func TestCalculatePartialMD5(t *testing.T) { - partialMD5, err := CalculatePartialMD5("../_test_files/alice.epub") + assert := assert.New(t) - want := "386d1cb51fe4a72e5c9fdad5e059bad9" - if partialMD5 != want { - t.Fatalf(`Expected: %v, Got: %v, Error: %v`, want, partialMD5, err) - } + desiredPartialMD5 := "386d1cb51fe4a72e5c9fdad5e059bad9" + calculatedPartialMD5, err := CalculatePartialMD5("../_test_files/alice.epub") + + assert.Nil(err, "error should be nil") + assert.Equal(desiredPartialMD5, *calculatedPartialMD5, "should be equal") +} + +func TestCalculateMD5(t *testing.T) { + assert := assert.New(t) + + desiredMD5 := "0f36c66155de34b281c4791654d0b1ce" + calculatedMD5, err := CalculateMD5("../_test_files/alice.epub") + + assert.Nil(err, "error should be nil") + assert.Equal(desiredMD5, *calculatedMD5, "should be equal") }