evan/AnthoLume
evan
/
AnthoLume
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

tests(all): improve tests, refactor(api): saving books
continuous-integration/drone/push Build is passing Details

This commit is contained in:
Evan Reichard 2024-02-24 20:45:26 -05:00
parent 803c187a00
commit 75ed394f8d
28 changed files with 1033 additions and 595 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
.drone.yml
View File

@ -4,21 +4,10 @@ name: default
steps:
# Unit Tests
- name: unit test
- name: tests
image: golang
commands:
- make tests_unit
# Integration Tests (Every Month)
- name: integration test
image: golang
commands:
- make tests_integration
when:
event:
- cron
cron:
- integration-test
- make tests
# Fetch tags
- name: fetch tags

1
.gitignore vendored
View File

@ -3,3 +3,4 @@ TODO.md
data/
build/
.direnv/
cover.html

9
Makefile
View File

@ -42,8 +42,7 @@ dev: build_tailwind
clean:
rm -rf ./build
tests_integration:
go test -v -tags=integration -coverpkg=./... ./metadata
tests_unit:
SET_TEST=set_val go test -v -coverpkg=./... ./...
tests:
SET_TEST=set_val go test -coverpkg=./... ./... -coverprofile=./cover.out
go tool cover -html=./cover.out -o ./cover.html
rm ./cover.out

90
api/app-admin-routes.go
View File

@ -19,6 +19,7 @@ import (
"github.com/gin-gonic/gin"
"github.com/itchyny/gojq"
log "github.com/sirupsen/logrus"
"reichard.io/antholume/metadata"
)
type adminAction string
@ -63,7 +64,7 @@ func (api *API) appPerformAdminAction(c *gin.Context) {
var rAdminAction requestAdminAction
if err := c.ShouldBind(&rAdminAction); err != nil {
log.Error("Invalid Form Bind: ", err)
appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values.")
appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values")
return
}
@ -75,6 +76,7 @@ func (api *API) appPerformAdminAction(c *gin.Context) {
// 2. Select all / deselect?
case adminCacheTables:
go api.db.CacheTempTables()
// TODO - Message
case adminRestore:
api.processRestoreFile(rAdminAction, c)
return
@ -83,7 +85,7 @@ func (api *API) appPerformAdminAction(c *gin.Context) {
_, err := api.db.DB.ExecContext(api.db.Ctx, "VACUUM;")
if err != nil {
log.Error("Unable to vacuum DB: ", err)
appErrorPage(c, http.StatusInternalServerError, "Unable to vacuum database.")
appErrorPage(c, http.StatusInternalServerError, "Unable to vacuum database")
return
}
@ -126,7 +128,7 @@ func (api *API) appGetAdminLogs(c *gin.Context) {
var rAdminLogs requestAdminLogs
if err := c.ShouldBindQuery(&rAdminLogs); err != nil {
log.Error("Invalid URI Bind")
appErrorPage(c, http.StatusNotFound, "Invalid URI parameters.")
appErrorPage(c, http.StatusNotFound, "Invalid URI parameters")
return
}
rAdminLogs.Filter = strings.TrimSpace(rAdminLogs.Filter)
@ -136,14 +138,14 @@ func (api *API) appGetAdminLogs(c *gin.Context) {
parsed, err := gojq.Parse(rAdminLogs.Filter)
if err != nil {
log.Error("Unable to parse JQ filter")
appErrorPage(c, http.StatusNotFound, "Unable to parse JQ filter.")
appErrorPage(c, http.StatusNotFound, "Unable to parse JQ filter")
return
}
jqFilter, err = gojq.Compile(parsed)
if err != nil {
log.Error("Unable to compile JQ filter")
appErrorPage(c, http.StatusNotFound, "Unable to compile JQ filter.")
appErrorPage(c, http.StatusNotFound, "Unable to compile JQ filter")
return
}
}
@ -152,7 +154,7 @@ func (api *API) appGetAdminLogs(c *gin.Context) {
logPath := filepath.Join(api.cfg.ConfigPath, "logs/antholume.log")
logFile, err := os.Open(logPath)
if err != nil {
appErrorPage(c, http.StatusBadRequest, "Missing AnthoLume log file.")
appErrorPage(c, http.StatusBadRequest, "Missing AnthoLume log file")
return
}
defer logFile.Close()
@ -229,7 +231,7 @@ func (api *API) appGetAdminImport(c *gin.Context) {
var rImportFolder requestAdminImport
if err := c.ShouldBindQuery(&rImportFolder); err != nil {
log.Error("Invalid URI Bind")
appErrorPage(c, http.StatusNotFound, "Invalid directory.")
appErrorPage(c, http.StatusNotFound, "Invalid directory")
return
}
@ -244,7 +246,7 @@ func (api *API) appGetAdminImport(c *gin.Context) {
dPath, err := filepath.Abs(api.cfg.DataPath)
if err != nil {
log.Error("Absolute filepath error: ", rImportFolder.Directory)
appErrorPage(c, http.StatusNotFound, "Unable to get data directory absolute path.")
appErrorPage(c, http.StatusNotFound, "Unable to get data directory absolute path")
return
}
@ -254,7 +256,7 @@ func (api *API) appGetAdminImport(c *gin.Context) {
entries, err := os.ReadDir(rImportFolder.Directory)
if err != nil {
log.Error("Invalid directory: ", rImportFolder.Directory)
appErrorPage(c, http.StatusNotFound, "Invalid directory.")
appErrorPage(c, http.StatusNotFound, "Invalid directory")
return
}
@ -279,13 +281,46 @@ func (api *API) appPerformAdminImport(c *gin.Context) {
var rAdminImport requestAdminImport
if err := c.ShouldBind(&rAdminImport); err != nil {
log.Error("Invalid URI Bind")
appErrorPage(c, http.StatusNotFound, "Invalid directory.")
appErrorPage(c, http.StatusNotFound, "Invalid directory")
return
}
// TODO
// TODO - Store results for approval?
fmt.Println(rAdminImport)
// Walk import directory & copy or import files
importDirectory := filepath.Clean(rAdminImport.Directory)
_ = filepath.WalkDir(importDirectory, func(currentPath string, f fs.DirEntry, err error) error {
if err != nil {
return err
}
if f.IsDir() {
return nil
}
// Get metadata
fileMeta, err := metadata.GetMetadata(currentPath)
if err != nil {
fmt.Printf("metadata error: %v\n", err)
return nil
}
// Only needed if copying
newName := deriveBaseFileName(fileMeta)
// Open File on Disk
// file, err := os.Open(currentPath)
// if err != nil {
// return err
// }
// defer file.Close()
// TODO - BasePath in DB
// TODO - Copy / Import
fmt.Printf("New File Metadata: %s\n", newName)
return nil
})
templateVars["CurrentPath"] = filepath.Clean(rAdminImport.Directory)
@ -297,14 +332,14 @@ func (api *API) processRestoreFile(rAdminAction requestAdminAction, c *gin.Conte
uploadedFile, err := rAdminAction.RestoreFile.Open()
if err != nil {
log.Error("File Error: ", err)
appErrorPage(c, http.StatusInternalServerError, "Unable to open file.")
appErrorPage(c, http.StatusInternalServerError, "Unable to open file")
return
}
fileMime, err := mimetype.DetectReader(uploadedFile)
if err != nil {
log.Error("MIME Error")
appErrorPage(c, http.StatusInternalServerError, "Unable to detect filetype.")
appErrorPage(c, http.StatusInternalServerError, "Unable to detect filetype")
return
}
fileExtension := fileMime.Extension()
@ -312,7 +347,7 @@ func (api *API) processRestoreFile(rAdminAction requestAdminAction, c *gin.Conte
// Validate Extension
if !slices.Contains([]string{".zip"}, fileExtension) {
log.Error("Invalid FileType: ", fileExtension)
appErrorPage(c, http.StatusBadRequest, "Invalid filetype.")
appErrorPage(c, http.StatusBadRequest, "Invalid filetype")
return
}
@ -320,7 +355,7 @@ func (api *API) processRestoreFile(rAdminAction requestAdminAction, c *gin.Conte
tempFile, err := os.CreateTemp("", "restore")
if err != nil {
log.Warn("Temp File Create Error: ", err)
appErrorPage(c, http.StatusInternalServerError, "Unable to create temp file.")
appErrorPage(c, http.StatusInternalServerError, "Unable to create temp file")
return
}
defer os.Remove(tempFile.Name())
@ -330,7 +365,7 @@ func (api *API) processRestoreFile(rAdminAction requestAdminAction, c *gin.Conte
err = c.SaveUploadedFile(rAdminAction.RestoreFile, tempFile.Name())
if err != nil {
log.Error("File Error: ", err)
appErrorPage(c, http.StatusInternalServerError, "Unable to save file.")
appErrorPage(c, http.StatusInternalServerError, "Unable to save file")
return
}
@ -338,7 +373,7 @@ func (api *API) processRestoreFile(rAdminAction requestAdminAction, c *gin.Conte
fileInfo, err := tempFile.Stat()
if err != nil {
log.Error("File Error: ", err)
appErrorPage(c, http.StatusInternalServerError, "Unable to read file.")
appErrorPage(c, http.StatusInternalServerError, "Unable to read file")
return
}
@ -346,7 +381,7 @@ func (api *API) processRestoreFile(rAdminAction requestAdminAction, c *gin.Conte
zipReader, err := zip.NewReader(tempFile, fileInfo.Size())
if err != nil {
log.Error("ZIP Error: ", err)
appErrorPage(c, http.StatusInternalServerError, "Unable to read zip.")
appErrorPage(c, http.StatusInternalServerError, "Unable to read zip")
return
}
@ -380,7 +415,7 @@ func (api *API) processRestoreFile(rAdminAction requestAdminAction, c *gin.Conte
backupFile, err := os.Create(backupFilePath)
if err != nil {
log.Error("Unable to create backup file: ", err)
appErrorPage(c, http.StatusInternalServerError, "Unable to create backup file.")
appErrorPage(c, http.StatusInternalServerError, "Unable to create backup file")
return
}
defer backupFile.Close()
@ -389,7 +424,7 @@ func (api *API) processRestoreFile(rAdminAction requestAdminAction, c *gin.Conte
_, err = api.db.DB.ExecContext(api.db.Ctx, "VACUUM;")
if err != nil {
log.Error("Unable to vacuum DB: ", err)
appErrorPage(c, http.StatusInternalServerError, "Unable to vacuum database.")
appErrorPage(c, http.StatusInternalServerError, "Unable to vacuum database")
return
}
@ -398,7 +433,7 @@ func (api *API) processRestoreFile(rAdminAction requestAdminAction, c *gin.Conte
err = api.createBackup(w, []string{"covers", "documents"})
if err != nil {
log.Error("Unable to save backup file: ", err)
appErrorPage(c, http.StatusInternalServerError, "Unable to save backup file.")
appErrorPage(c, http.StatusInternalServerError, "Unable to save backup file")
return
}
@ -406,26 +441,26 @@ func (api *API) processRestoreFile(rAdminAction requestAdminAction, c *gin.Conte
err = api.removeData()
if err != nil {
log.Error("Unable to delete data: ", err)
appErrorPage(c, http.StatusInternalServerError, "Unable to delete data.")
appErrorPage(c, http.StatusInternalServerError, "Unable to delete data")
return
}
// Restore Data
err = api.restoreData(zipReader)
if err != nil {
appErrorPage(c, http.StatusInternalServerError, "Unable to restore data.")
appErrorPage(c, http.StatusInternalServerError, "Unable to restore data")
log.Panic("Unable to restore data: ", err)
}
// Reinit DB
if err := api.db.Reload(); err != nil {
appErrorPage(c, http.StatusInternalServerError, "Unable to reload DB.")
appErrorPage(c, http.StatusInternalServerError, "Unable to reload DB")
log.Panicf("Unable to reload DB: %v", err)
}
// Rotate Auth Hashes
if err := api.rotateAllAuthHashes(); err != nil {
appErrorPage(c, http.StatusInternalServerError, "Unable to rotate hashes.")
appErrorPage(c, http.StatusInternalServerError, "Unable to rotate hashes")
log.Panicf("Unable to rotate auth hashes: %v", err)
}
@ -433,6 +468,7 @@ func (api *API) processRestoreFile(rAdminAction requestAdminAction, c *gin.Conte
c.Redirect(http.StatusFound, "/login")
}
// Restore all data
func (api *API) restoreData(zipReader *zip.Reader) error {
// Ensure Directories
api.cfg.EnsureDirectories()
@ -463,6 +499,7 @@ func (api *API) restoreData(zipReader *zip.Reader) error {
return nil
}
// Remove all data
func (api *API) removeData() error {
allPaths := []string{
"covers",
@ -485,6 +522,7 @@ func (api *API) removeData() error {
return nil
}
// Backup all data
func (api *API) createBackup(w io.Writer, directories []string) error {
ar := zip.NewWriter(w)

157
api/app-routes.go
View File

@ -157,7 +157,7 @@ func (api *API) appGetDocument(c *gin.Context) {
var rDocID requestDocumentID
if err := c.ShouldBindUri(&rDocID); err != nil {
log.Error("Invalid URI Bind")
appErrorPage(c, http.StatusNotFound, "Invalid document.")
appErrorPage(c, http.StatusNotFound, "Invalid document")
return
}
@ -361,7 +361,7 @@ func (api *API) appGetDocumentProgress(c *gin.Context) {
var rDoc requestDocumentID
if err := c.ShouldBindUri(&rDoc); err != nil {
log.Error("Invalid URI Bind")
appErrorPage(c, http.StatusNotFound, "Invalid document.")
appErrorPage(c, http.StatusNotFound, "Invalid document")
return
}
@ -417,7 +417,7 @@ func (api *API) appUploadNewDocument(c *gin.Context) {
var rDocUpload requestDocumentUpload
if err := c.ShouldBind(&rDocUpload); err != nil {
log.Error("Invalid Form Bind")
appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values.")
appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values")
return
}
@ -426,153 +426,92 @@ func (api *API) appUploadNewDocument(c *gin.Context) {
return
}
// Validate Type & Derive Extension on MIME
uploadedFile, err := rDocUpload.DocumentFile.Open()
if err != nil {
log.Error("File Error: ", err)
appErrorPage(c, http.StatusInternalServerError, "Unable to open file.")
return
}
fileMime, err := mimetype.DetectReader(uploadedFile)
if err != nil {
log.Error("MIME Error")
appErrorPage(c, http.StatusInternalServerError, "Unable to detect filetype.")
return
}
fileExtension := fileMime.Extension()
// Validate Extension
if !slices.Contains([]string{".epub"}, fileExtension) {
log.Error("Invalid FileType: ", fileExtension)
appErrorPage(c, http.StatusBadRequest, "Invalid filetype.")
return
}
// Create Temp File
tempFile, err := os.CreateTemp("", "book")
if err != nil {
log.Warn("Temp File Create Error: ", err)
appErrorPage(c, http.StatusInternalServerError, "Unable to create temp file.")
appErrorPage(c, http.StatusInternalServerError, "Unable to create temp file")
return
}
defer os.Remove(tempFile.Name())
defer tempFile.Close()
// Save Temp
// Save Temp File
err = c.SaveUploadedFile(rDocUpload.DocumentFile, tempFile.Name())
if err != nil {
log.Error("File Error: ", err)
appErrorPage(c, http.StatusInternalServerError, "Unable to save file.")
appErrorPage(c, http.StatusInternalServerError, "Unable to save file")
return
}
// Get Metadata
metadataInfo, err := metadata.GetMetadata(tempFile.Name())
if err != nil {
log.Warn("GetMetadata Error: ", err)
appErrorPage(c, http.StatusInternalServerError, "Unable to acquire file metadata.")
log.Errorf("unable to acquire metadata: %v", err)
appErrorPage(c, http.StatusInternalServerError, "Unable to acquire metadata")
return
}
// Calculate Partial MD5 ID
partialMD5, err := utils.CalculatePartialMD5(tempFile.Name())
if err != nil {
log.Warn("Partial MD5 Error: ", err)
appErrorPage(c, http.StatusInternalServerError, "Unable to calculate partial MD5.")
return
}
// Check Exists
_, err = api.db.Queries.GetDocument(api.db.Ctx, partialMD5)
// Check Already Exists
_, err = api.db.Queries.GetDocument(api.db.Ctx, *metadataInfo.PartialMD5)
if err == nil {
c.Redirect(http.StatusFound, fmt.Sprintf("./documents/%s", partialMD5))
return
log.Warnf("document already exists: %s", *metadataInfo.PartialMD5)
c.Redirect(http.StatusFound, fmt.Sprintf("./documents/%s", *metadataInfo.PartialMD5))
}
// Calculate Actual MD5
fileHash, err := getFileMD5(tempFile.Name())
if err != nil {
log.Error("MD5 Hash Failure: ", err)
appErrorPage(c, http.StatusInternalServerError, "Unable to calculate MD5.")
return
}
// Get Word Count
wordCount, err := metadata.GetWordCount(tempFile.Name())
if err != nil {
log.Error("Word Count Failure: ", err)
appErrorPage(c, http.StatusInternalServerError, "Unable to calculate word count.")
return
}
// Derive Filename
var fileName string
if *metadataInfo.Author != "" {
fileName = fileName + *metadataInfo.Author
} else {
fileName = fileName + "Unknown"
}
if *metadataInfo.Title != "" {
fileName = fileName + " - " + *metadataInfo.Title
} else {
fileName = fileName + " - Unknown"
}
// Remove Slashes
fileName = strings.ReplaceAll(fileName, "/", "")
// Derive & Sanitize File Name
fileName = "." + filepath.Clean(fmt.Sprintf("/%s [%s]%s", fileName, partialMD5, fileExtension))
// Generate Storage Path & Open File
fileName := deriveBaseFileName(metadataInfo)
safePath := filepath.Join(api.cfg.DataPath, "documents", fileName)
// Open Destination File
destFile, err := os.Create(safePath)
if err != nil {
log.Error("Dest File Error: ", err)
appErrorPage(c, http.StatusInternalServerError, "Unable to save file.")
log.Errorf("unable to open destination file: %v", err)
appErrorPage(c, http.StatusInternalServerError, "Unable to open destination file")
return
}
defer destFile.Close()
// Copy File
if _, err = io.Copy(destFile, tempFile); err != nil {
log.Error("Copy Temp File Error: ", err)
appErrorPage(c, http.StatusInternalServerError, "Unable to save file.")
log.Errorf("unable to save file: %v", err)
appErrorPage(c, http.StatusInternalServerError, "Unable to save file")
return
}
// Upsert Document
if _, err = api.db.Queries.UpsertDocument(api.db.Ctx, database.UpsertDocumentParams{
ID: partialMD5,
ID: *metadataInfo.PartialMD5,
Title: metadataInfo.Title,
Author: metadataInfo.Author,
Description: metadataInfo.Description,
Words: &wordCount,
Md5: fileHash,
Md5: metadataInfo.MD5,
Words: metadataInfo.WordCount,
Filepath: &fileName,
// TODO (BasePath):
// - Should be current config directory
}); err != nil {
log.Error("UpsertDocument DB Error: ", err)
log.Errorf("UpsertDocument DB Error: %v", err)
appErrorPage(c, http.StatusInternalServerError, fmt.Sprintf("UpsertDocument DB Error: %v", err))
return
}
c.Redirect(http.StatusFound, fmt.Sprintf("./documents/%s", partialMD5))
c.Redirect(http.StatusFound, fmt.Sprintf("./documents/%s", *metadataInfo.PartialMD5))
}
func (api *API) appEditDocument(c *gin.Context) {
var rDocID requestDocumentID
if err := c.ShouldBindUri(&rDocID); err != nil {
log.Error("Invalid URI Bind")
appErrorPage(c, http.StatusNotFound, "Invalid document.")
appErrorPage(c, http.StatusNotFound, "Invalid document")
return
}
var rDocEdit requestDocumentEdit
if err := c.ShouldBind(&rDocEdit); err != nil {
log.Error("Invalid Form Bind")
appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values.")
appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values")
return
}
@ -586,7 +525,7 @@ func (api *API) appEditDocument(c *gin.Context) {
rDocEdit.CoverGBID == nil &&
rDocEdit.CoverFile == nil {
log.Error("Missing Form Values")
appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values.")
appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values")
return
}
@ -600,14 +539,14 @@ func (api *API) appEditDocument(c *gin.Context) {
uploadedFile, err := rDocEdit.CoverFile.Open()
if err != nil {
log.Error("File Error")
appErrorPage(c, http.StatusInternalServerError, "Unable to open file.")
appErrorPage(c, http.StatusInternalServerError, "Unable to open file")
return
}
fileMime, err := mimetype.DetectReader(uploadedFile)
if err != nil {
log.Error("MIME Error")
appErrorPage(c, http.StatusInternalServerError, "Unable to detect filetype.")
appErrorPage(c, http.StatusInternalServerError, "Unable to detect filetype")
return
}
fileExtension := fileMime.Extension()
@ -615,7 +554,7 @@ func (api *API) appEditDocument(c *gin.Context) {
// Validate Extension
if !slices.Contains([]string{".jpg", ".png"}, fileExtension) {
log.Error("Invalid FileType: ", fileExtension)
appErrorPage(c, http.StatusBadRequest, "Invalid filetype.")
appErrorPage(c, http.StatusBadRequest, "Invalid filetype")
return
}
@ -627,7 +566,7 @@ func (api *API) appEditDocument(c *gin.Context) {
err = c.SaveUploadedFile(rDocEdit.CoverFile, safePath)
if err != nil {
log.Error("File Error: ", err)
appErrorPage(c, http.StatusInternalServerError, "Unable to save file.")
appErrorPage(c, http.StatusInternalServerError, "Unable to save file")
return
}
@ -663,7 +602,7 @@ func (api *API) appDeleteDocument(c *gin.Context) {
var rDocID requestDocumentID
if err := c.ShouldBindUri(&rDocID); err != nil {
log.Error("Invalid URI Bind")
appErrorPage(c, http.StatusNotFound, "Invalid document.")
appErrorPage(c, http.StatusNotFound, "Invalid document")
return
}
changed, err := api.db.Queries.DeleteDocument(api.db.Ctx, rDocID.DocumentID)
@ -674,7 +613,7 @@ func (api *API) appDeleteDocument(c *gin.Context) {
}
if changed == 0 {
log.Error("DeleteDocument DB Error")
appErrorPage(c, http.StatusNotFound, "Invalid document.")
appErrorPage(c, http.StatusNotFound, "Invalid document")
return
}
@ -685,14 +624,14 @@ func (api *API) appIdentifyDocument(c *gin.Context) {
var rDocID requestDocumentID
if err := c.ShouldBindUri(&rDocID); err != nil {
log.Error("Invalid URI Bind")
appErrorPage(c, http.StatusNotFound, "Invalid document.")
appErrorPage(c, http.StatusNotFound, "Invalid document")
return
}
var rDocIdentify requestDocumentIdentify
if err := c.ShouldBind(&rDocIdentify); err != nil {
log.Error("Invalid Form Bind")
appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values.")
appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values")
return
}
@ -710,7 +649,7 @@ func (api *API) appIdentifyDocument(c *gin.Context) {
// Validate Values
if rDocIdentify.ISBN == nil && rDocIdentify.Title == nil && rDocIdentify.Author == nil {
log.Error("Invalid Form")
appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values.")
appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values")
return
}
@ -718,7 +657,7 @@ func (api *API) appIdentifyDocument(c *gin.Context) {
templateVars, auth := api.getBaseTemplateVars("document", c)
// Get Metadata
metadataResults, err := metadata.SearchMetadata(metadata.GBOOK, metadata.MetadataInfo{
metadataResults, err := metadata.SearchMetadata(metadata.SOURCE_GBOOK, metadata.MetadataInfo{
Title: rDocIdentify.Title,
Author: rDocIdentify.Author,
ISBN10: rDocIdentify.ISBN,
@ -767,7 +706,7 @@ func (api *API) appSaveNewDocument(c *gin.Context) {
var rDocAdd requestDocumentAdd
if err := c.ShouldBind(&rDocAdd); err != nil {
log.Error("Invalid Form Bind")
appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values.")
appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values")
return
}
@ -845,7 +784,7 @@ func (api *API) appSaveNewDocument(c *gin.Context) {
fileName = strings.ReplaceAll(fileName, "/", "")
// Derive & Sanitize File Name
fileName = "." + filepath.Clean(fmt.Sprintf("/%s [%s]%s", fileName, partialMD5, fileExtension))
fileName = "." + filepath.Clean(fmt.Sprintf("/%s [%s]%s", fileName, *partialMD5, fileExtension))
// Open Source File
sourceFile, err := os.Open(tempFilePath)
@ -901,12 +840,12 @@ func (api *API) appSaveNewDocument(c *gin.Context) {
// Upsert Document
if _, err = api.db.Queries.UpsertDocument(api.db.Ctx, database.UpsertDocumentParams{
ID: partialMD5,
ID: *partialMD5,
Title: rDocAdd.Title,
Author: rDocAdd.Author,
Md5: fileHash,
Filepath: &fileName,
Words: &wordCount,
Words: wordCount,
}); err != nil {
log.Error("UpsertDocument DB Error: ", err)
sendDownloadMessage("Unable to save to database", gin.H{"Error": true})
@ -917,7 +856,7 @@ func (api *API) appSaveNewDocument(c *gin.Context) {
sendDownloadMessage("Download Success", gin.H{
"Progress": 100,
"ButtonText": "Go to Book",
"ButtonHref": fmt.Sprintf("./documents/%s", partialMD5),
"ButtonHref": fmt.Sprintf("./documents/%s", *partialMD5),
})
}
@ -925,14 +864,14 @@ func (api *API) appEditSettings(c *gin.Context) {
var rUserSettings requestSettingsEdit
if err := c.ShouldBind(&rUserSettings); err != nil {
log.Error("Invalid Form Bind")
appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values.")
appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values")
return
}
// Validate Something Exists
if rUserSettings.Password == nil && rUserSettings.NewPassword == nil && rUserSettings.TimeOffset == nil {
log.Error("Missing Form Values")
appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values.")
appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values")
return
}
@ -1023,7 +962,7 @@ func (api *API) getDocumentsWordCount(documents []database.GetDocumentsWithStats
} else {
if _, err := qtx.UpsertDocument(api.db.Ctx, database.UpsertDocumentParams{
ID: item.ID,
Words: &wordCount,
Words: wordCount,
}); err != nil {
log.Error("UpsertDocument DB Error: ", err)
return err

2
api/common.go
View File

@ -95,7 +95,7 @@ func (api *API) createGetCoverHandler(errorFunc func(*gin.Context, int, string))
var coverFile string = "UNKNOWN"
// Identify Documents & Save Covers
metadataResults, err := metadata.SearchMetadata(metadata.GBOOK, metadata.MetadataInfo{
metadataResults, err := metadata.SearchMetadata(metadata.SOURCE_GBOOK, metadata.MetadataInfo{
Title: document.Title,
Author: document.Author,
})

77
api/ko-routes.go
View File

@ -10,13 +10,10 @@ import (
"net/http"
"os"
"path/filepath"
"strings"
"time"
"github.com/gabriel-vasile/mimetype"
"github.com/gin-gonic/gin"
log "github.com/sirupsen/logrus"
"golang.org/x/exp/slices"
"reichard.io/antholume/database"
"reichard.io/antholume/metadata"
)
@ -456,21 +453,11 @@ func (api *API) koUploadExistingDocument(c *gin.Context) {
return
}
// Open Form File
fileData, err := c.FormFile("file")
if err != nil {
log.Error("File Error:", err)
apiErrorPage(c, http.StatusBadRequest, "File Error")
return
}
// Validate Type & Derive Extension on MIME
uploadedFile, err := fileData.Open()
fileMime, err := mimetype.DetectReader(uploadedFile)
fileExtension := fileMime.Extension()
if !slices.Contains([]string{".epub", ".html"}, fileExtension) {
log.Error("Invalid FileType:", fileExtension)
apiErrorPage(c, http.StatusBadRequest, "Invalid Filetype")
apiErrorPage(c, http.StatusBadRequest, "File error")
return
}
@ -482,25 +469,29 @@ func (api *API) koUploadExistingDocument(c *gin.Context) {
return
}
// Open File
uploadedFile, err := fileData.Open()
if err != nil {
log.Error("Unable to open file")
apiErrorPage(c, http.StatusBadRequest, "Unable to open file")
return
}
// Check Support
docType, err := metadata.GetDocumentTypeReader(uploadedFile)
if err != nil {
log.Error("Unsupported file")
apiErrorPage(c, http.StatusBadRequest, "Unsupported file")
return
}
// Derive Filename
var fileName string
if document.Author != nil {
fileName = fileName + *document.Author
} else {
fileName = fileName + "Unknown"
}
if document.Title != nil {
fileName = fileName + " - " + *document.Title
} else {
fileName = fileName + " - Unknown"
}
// Remove Slashes
fileName = strings.ReplaceAll(fileName, "/", "")
// Derive & Sanitize File Name
fileName = "." + filepath.Clean(fmt.Sprintf("/%s [%s]%s", fileName, document.ID, fileExtension))
fileName := deriveBaseFileName(&metadata.MetadataInfo{
Type: *docType,
PartialMD5: &document.ID,
Title: document.Title,
Author: document.Author,
})
// Generate Storage Path
safePath := filepath.Join(api.cfg.DataPath, "documents", fileName)
@ -516,28 +507,20 @@ func (api *API) koUploadExistingDocument(c *gin.Context) {
}
}
// Get MD5 Hash
fileHash, err := getFileMD5(safePath)
// Acquire Metadata
metadataInfo, err := metadata.GetMetadata(safePath)
if err != nil {
log.Error("Hash Failure:", err)
apiErrorPage(c, http.StatusBadRequest, "File Error")
return
}
// Get Word Count
wordCount, err := metadata.GetWordCount(safePath)
if err != nil {
log.Error("Word Count Failure:", err)
apiErrorPage(c, http.StatusBadRequest, "File Error")
log.Errorf("Unable to acquire metadata: %v", err)
apiErrorPage(c, http.StatusBadRequest, "Unable to acquire metadata")
return
}
// Upsert Document
if _, err = api.db.Queries.UpsertDocument(api.db.Ctx, database.UpsertDocumentParams{
ID: document.ID,
Md5: fileHash,
Md5: metadataInfo.MD5,
Words: metadataInfo.WordCount,
Filepath: &fileName,
Words: &wordCount,
}); err != nil {
log.Error("UpsertDocument DB Error:", err)
apiErrorPage(c, http.StatusBadRequest, "Document Error")

22
api/utils.go
View File

@ -4,10 +4,13 @@ import (
"errors"
"fmt"
"math"
"path/filepath"
"reflect"
"strings"
"reichard.io/antholume/database"
"reichard.io/antholume/graph"
"reichard.io/antholume/metadata"
)
type UTCOffset struct {
@ -144,3 +147,22 @@ func fields(value interface{}) (map[string]interface{}, error) {
}
return m, nil
}
func deriveBaseFileName(metadataInfo *metadata.MetadataInfo) string {
// Derive New FileName
var newFileName string
if *metadataInfo.Author != "" {
newFileName = newFileName + *metadataInfo.Author
} else {
newFileName = newFileName + "Unknown"
}
if *metadataInfo.Title != "" {
newFileName = newFileName + " - " + *metadataInfo.Title
} else {
newFileName = newFileName + " - Unknown"
}
// Remove Slashes
fileName := strings.ReplaceAll(newFileName, "/", "")
return "." + filepath.Clean(fmt.Sprintf("/%s [%s]%s", fileName, *metadataInfo.PartialMD5, metadataInfo.Type))
}

35
api/utils_test.go
View File

@ -1,12 +1,35 @@
package api
import "testing"
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestNiceSeconds(t *testing.T) {
want := "22d 7h 39m 31s"
nice := niceSeconds(1928371)
wantOne := "22d 7h 39m 31s"
wantNA := "N/A"
if nice != want {
t.Fatalf(`Expected: %v, Got: %v`, want, nice)
}
niceOne := niceSeconds(1928371)
niceNA := niceSeconds(0)
assert.Equal(t, wantOne, niceOne, "should be nice seconds")
assert.Equal(t, wantNA, niceNA, "should be nice NA")
}
func TestNiceNumbers(t *testing.T) {
wantMillions := "198M"
wantThousands := "19.8k"
wantThousandsTwo := "1.98k"
wantZero := "0"
niceMillions := niceNumbers(198236461)
niceThousands := niceNumbers(19823)
niceThousandsTwo := niceNumbers(1984)
niceZero := niceNumbers(0)
assert.Equal(t, wantMillions, niceMillions, "should be nice millions")
assert.Equal(t, wantThousands, niceThousands, "should be nice thousands")
assert.Equal(t, wantThousandsTwo, niceThousandsTwo, "should be nice thousands")
assert.Equal(t, wantZero, niceZero, "should be nice zero")
}

1
config/config.go
View File

@ -118,6 +118,7 @@ func (c *Config) EnsureDirectories() {
docDir := filepath.Join(c.DataPath, "documents")
coversDir := filepath.Join(c.DataPath, "covers")
backupDir := filepath.Join(c.DataPath, "backups")
os.Mkdir(docDir, 0755)
os.Mkdir(coversDir, 0755)
os.Mkdir(backupDir, 0755)

46
config/config_test.go
View File

@ -1,35 +1,37 @@
package config
import "testing"
import (
"runtime"
"testing"
"github.com/stretchr/testify/assert"
)
func TestLoadConfig(t *testing.T) {
conf := Load()
want := "sqlite"
if conf.DBType != want {
t.Fatalf(`Load().DBType = %q, want match for %#q, nil`, conf.DBType, want)
}
assert.Equal(t, "sqlite", conf.DBType)
}
func TestGetEnvDefault(t *testing.T) {
want := "def_val"
envDefault := getEnv("DEFAULT_TEST", want)
if envDefault != want {
t.Fatalf(`getEnv("DEFAULT_TEST", "def_val") = %q, want match for %#q, nil`, envDefault, want)
}
}
desiredValue := "def_val"
envDefault := getEnv("DEFAULT_TEST", desiredValue)
func TestGetEnvSet(t *testing.T) {
envDefault := getEnv("SET_TEST", "not_this")
want := "set_val"
if envDefault != want {
t.Fatalf(`getEnv("SET_TEST", "not_this") = %q, want match for %#q, nil`, envDefault, want)
}
assert.Equal(t, desiredValue, envDefault)
}
func TestTrimLowerString(t *testing.T) {
want := "trimtest"
output := trimLowerString(" trimTest ")
if output != want {
t.Fatalf(`trimLowerString(" trimTest ") = %q, want match for %#q, nil`, output, want)
}
desiredValue := "trimtest"
outputValue := trimLowerString(" trimTest ")
assert.Equal(t, desiredValue, outputValue)
}
func TestPrettyCaller(t *testing.T) {
p, _, _, _ := runtime.Caller(0)
result := runtime.CallersFrames([]uintptr{p})
f, _ := result.Next()
functionName, fileName := prettyCaller(&f)
assert.Equal(t, "TestPrettyCaller", functionName, "should have current function name")
assert.Equal(t, "config/config_test.go@30", fileName, "should have current file path and line number")
}

20
database/manager_test.go
View File

@ -5,6 +5,7 @@ import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"reichard.io/antholume/config"
"reichard.io/antholume/utils"
)
@ -28,9 +29,7 @@ func TestNewMgr(t *testing.T) {
}
dbm := NewMgr(&cfg)
if dbm == nil {
t.Fatalf(`Expected: *DBManager, Got: nil`)
}
assert.NotNil(t, dbm, "should not be nil dbm")
t.Run("Database", func(t *testing.T) {
dt := databaseTest{t, dbm}
@ -46,9 +45,7 @@ func (dt *databaseTest) TestUser() {
dt.Run("User", func(t *testing.T) {
// Generate Auth Hash
rawAuthHash, err := utils.GenerateToken(64)
if err != nil {
t.Fatalf(`Expected: %v, Got: %v, Error: %v`, nil, err, err)
}
assert.Nil(t, err, "should be nil err")
authHash := fmt.Sprintf("%x", rawAuthHash)
changed, err := dt.dbm.Queries.CreateUser(dt.dbm.Ctx, CreateUserParams{
@ -57,14 +54,13 @@ func (dt *databaseTest) TestUser() {
AuthHash: &authHash,
})
if err != nil || changed != 1 {
t.Fatalf(`Expected: %v, Got: %v, Error: %v`, 1, changed, err)
}
assert.Nil(t, err, "should be nil err")
assert.Equal(t, int64(1), changed)
user, err := dt.dbm.Queries.GetUser(dt.dbm.Ctx, userID)
if err != nil || *user.Pass != userPass {
t.Fatalf(`Expected: %v, Got: %v, Error: %v`, userPass, *user.Pass, err)
}
assert.Nil(t, err, "should be nil err")
assert.Equal(t, userPass, *user.Pass)
})
}

4
go.mod
View File

@ -27,6 +27,7 @@ require (
github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect
github.com/chenzhuoyu/iasm v0.9.1 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect
github.com/go-playground/locales v0.14.1 // indirect
@ -39,6 +40,7 @@ require (
github.com/gorilla/securecookie v1.1.2 // indirect
github.com/gorilla/sessions v1.2.2 // indirect
github.com/itchyny/timefmt-go v0.1.5 // indirect
github.com/jarcoal/httpmock v1.3.1 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
github.com/klauspost/cpuid/v2 v2.2.6 // indirect
@ -47,9 +49,11 @@ require (
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/pelletier/go-toml/v2 v2.1.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/sethvargo/go-retry v0.2.4 // indirect
github.com/stretchr/testify v1.8.4 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/ugorji/go/codec v1.2.12 // indirect
github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e // indirect

2
go.sum
View File

@ -127,6 +127,8 @@ github.com/jackc/pgx/v5 v5.5.1 h1:5I9etrGkLrN+2XPCsi6XLlV5DITbSL/xBZdmAxFcXPI=
github.com/jackc/pgx/v5 v5.5.1/go.mod h1:Ig06C2Vu0t5qXC60W8sqIthScaEnFvojjj9dSljmHRA=
github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/jarcoal/httpmock v1.3.1 h1:iUx3whfZWVf3jT01hQTO/Eo5sAYtB2/rqaUuOtpInww=
github.com/jarcoal/httpmock v1.3.1/go.mod h1:3yb8rc4BI7TCBhFY8ng0gjuLKJNquuDNiPaZjnENuYg=
github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4=
github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak=
github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4=

110
metadata/_test_files/gbooks_id_response.json Normal file
View File

@ -0,0 +1,110 @@
{
"kind": "books#volume",
"id": "ZxwpakTv_MIC",
"etag": "mhqr3GsebaQ",
"selfLink": "https://www.googleapis.com/books/v1/volumes/ZxwpakTv_MIC",
"volumeInfo": {
"title": "Alice in Wonderland",
"authors": [
"Lewis Carroll"
],
"publisher": "The Floating Press",
"publishedDate": "2009-01-01",
"description": "Alice in Wonderland (also known as Alice's Adventures in Wonderland), from 1865, is the peculiar and imaginative tale of a girl who falls down a rabbit-hole into a bizarre world of eccentric and unusual creatures. Lewis Carroll's prominent example of the genre of \"literary nonsense\" has endured in popularity with its clever way of playing with logic and a narrative structure that has influence generations of fiction writing.",
"industryIdentifiers": [
{
"type": "ISBN_10",
"identifier": "1877527815"
},
{
"type": "ISBN_13",
"identifier": "9781877527814"
}
],
"readingModes": {
"text": true,
"image": false
},
"pageCount": 104,
"printedPageCount": 112,
"printType": "BOOK",
"categories": [
"Fiction / Classics",
"Juvenile Fiction / General"
],
"averageRating": 5,
"ratingsCount": 1,
"maturityRating": "NOT_MATURE",
"allowAnonLogging": true,
"contentVersion": "0.2.3.0.preview.2",
"panelizationSummary": {
"containsEpubBubbles": false,
"containsImageBubbles": false
},
"imageLinks": {
"smallThumbnail": "http://books.google.com/books/content?id=ZxwpakTv_MIC&printsec=frontcover&img=1&zoom=5&edge=curl&imgtk=AFLRE71e5b-TeAKTiPSvXNUPeUi8rItzur2xSzwH8QU3qjKH0A2opmoq1o5I9RqJFt1BtcCCqILhnYRcB2aFLJmEvom11gx3Qn3PNN1iBLj2H5y2JHjM8wIwGT7iWFQmEn0Od7s6sOdk&source=gbs_api",
"thumbnail": "http://books.google.com/books/content?id=ZxwpakTv_MIC&printsec=frontcover&img=1&zoom=1&edge=curl&imgtk=AFLRE70QORt9J_DmKJgfyf9UEjQkdDMZ0qAu0GP315a1Q4CRS3snEjKnJJO2fYFdxjMwsSpmHoXDFPZbsy4gw-kMvF7lL8LtwxGbJGlfETHw_jbQBKBlKTrneK4XFvvV-EXNrZRgylxj&source=gbs_api",
"small": "http://books.google.com/books/content?id=ZxwpakTv_MIC&printsec=frontcover&img=1&zoom=2&edge=curl&imgtk=AFLRE70r1pAUt6VhuEEW8vXFhu8LvKln3yj0mdlaWPO4ZQuODLFQnH0fTebKMMX4ANR5i4PtC0oaI48XkwF-EdzlEM1WmUcR5383N4kRMXcta_i9nmb2y38dnh3hObwQW5VoAxbc9psn&source=gbs_api",
"medium": "http://books.google.com/books/content?id=ZxwpakTv_MIC&printsec=frontcover&img=1&zoom=3&edge=curl&imgtk=AFLRE7019EVuXvhzbhmtbz1QFh-ajB6kTKRHGhqijFf8big_GPRMMdpCdKlklFbkCfXvy8F64t5NKlThUHb3tFP-51bbDXkrVErFbCqKGzGnDSSm8cewqT8HiYDNHqn0hXYnuYvN4vYf&source=gbs_api",
"large": "http://books.google.com/books/content?id=ZxwpakTv_MIC&printsec=frontcover&img=1&zoom=4&edge=curl&imgtk=AFLRE72I15XZqp_8c8BAj4EskxkdC6nQz8F0Fs6VJhkykwIqfjzwuM34tUSQa3UnMGbx-UYjZjSLmCNFlePS8aR7yy-0UP9BRnYD-h5Qbesnnt_xdOb3u7Wdiobi6VbciNCBwUwbCyeH&source=gbs_api",
"extraLarge": "http://books.google.com/books/content?id=ZxwpakTv_MIC&printsec=frontcover&img=1&zoom=6&edge=curl&imgtk=AFLRE70rC6ktY6U0K_hqG1HxPl_9hMjpKb10p9DryVIwQgUjoJfWQOjpNA3EQ-5yk167yYDlO27gylqNAdJBYWu7ZHr3GuqkjTDpXjDvzBBppVyWaVNxKwhOz3gfJ-gzM6cC4kLHP26R&source=gbs_api"
},
"language": "en",
"previewLink": "http://books.google.com/books?id=ZxwpakTv_MIC&hl=&source=gbs_api",
"infoLink": "https://play.google.com/store/books/details?id=ZxwpakTv_MIC&source=gbs_api",
"canonicalVolumeLink": "https://play.google.com/store/books/details?id=ZxwpakTv_MIC"
},
"layerInfo": {
"layers": [
{
"layerId": "geo",
"volumeAnnotationsVersion": "2"
}
]
},
"saleInfo": {
"country": "US",
"saleability": "FOR_SALE",
"isEbook": true,
"listPrice": {
"amount": 3.99,
"currencyCode": "USD"
},
"retailPrice": {
"amount": 3.99,
"currencyCode": "USD"
},
"buyLink": "https://play.google.com/store/books/details?id=ZxwpakTv_MIC&rdid=book-ZxwpakTv_MIC&rdot=1&source=gbs_api",
"offers": [
{
"finskyOfferType": 1,
"listPrice": {
"amountInMicros": 3990000,
"currencyCode": "USD"
},
"retailPrice": {
"amountInMicros": 3990000,
"currencyCode": "USD"
},
"giftable": true
}
]
},
"accessInfo": {
"country": "US",
"viewability": "PARTIAL",
"embeddable": true,
"publicDomain": false,
"textToSpeechPermission": "ALLOWED",
"epub": {
"isAvailable": true,
"acsTokenLink": "http://books.google.com/books/download/Alice_in_Wonderland-sample-epub.acsm?id=ZxwpakTv_MIC&format=epub&output=acs4_fulfillment_token&dl_type=sample&source=gbs_api"
},
"pdf": {
"isAvailable": false
},
"webReaderLink": "http://play.google.com/books/reader?id=ZxwpakTv_MIC&hl=&source=gbs_api",
"accessViewStatus": "SAMPLE",
"quoteSharingAllowed": false
}
}

105
metadata/_test_files/gbooks_query_response.json Normal file
View File

@ -0,0 +1,105 @@
{
"kind": "books#volumes",
"totalItems": 1,
"items": [
{
"kind": "books#volume",
"id": "ZxwpakTv_MIC",
"etag": "F2eR9VV6VwQ",
"selfLink": "https://www.googleapis.com/books/v1/volumes/ZxwpakTv_MIC",
"volumeInfo": {
"title": "Alice in Wonderland",
"authors": [
"Lewis Carroll"
],
"publisher": "The Floating Press",
"publishedDate": "2009-01-01",
"description": "Alice in Wonderland (also known as Alice's Adventures in Wonderland), from 1865, is the peculiar and imaginative tale of a girl who falls down a rabbit-hole into a bizarre world of eccentric and unusual creatures. Lewis Carroll's prominent example of the genre of \"literary nonsense\" has endured in popularity with its clever way of playing with logic and a narrative structure that has influence generations of fiction writing.",
"industryIdentifiers": [
{
"type": "ISBN_13",
"identifier": "9781877527814"
},
{
"type": "ISBN_10",
"identifier": "1877527815"
}
],
"readingModes": {
"text": true,
"image": false
},
"pageCount": 104,
"printType": "BOOK",
"categories": [
"Fiction"
],
"averageRating": 5,
"ratingsCount": 1,
"maturityRating": "NOT_MATURE",
"allowAnonLogging": true,
"contentVersion": "0.2.3.0.preview.2",
"panelizationSummary": {
"containsEpubBubbles": false,
"containsImageBubbles": false
},
"imageLinks": {
"smallThumbnail": "http://books.google.com/books/content?id=ZxwpakTv_MIC&printsec=frontcover&img=1&zoom=5&edge=curl&source=gbs_api",
"thumbnail": "http://books.google.com/books/content?id=ZxwpakTv_MIC&printsec=frontcover&img=1&zoom=1&edge=curl&source=gbs_api"
},
"language": "en",
"previewLink": "http://books.google.com/books?id=ZxwpakTv_MIC&printsec=frontcover&dq=isbn:1877527815&hl=&cd=1&source=gbs_api",
"infoLink": "https://play.google.com/store/books/details?id=ZxwpakTv_MIC&source=gbs_api",
"canonicalVolumeLink": "https://play.google.com/store/books/details?id=ZxwpakTv_MIC"
},
"saleInfo": {
"country": "US",
"saleability": "FOR_SALE",
"isEbook": true,
"listPrice": {
"amount": 3.99,
"currencyCode": "USD"
},
"retailPrice": {
"amount": 3.99,
"currencyCode": "USD"
},
"buyLink": "https://play.google.com/store/books/details?id=ZxwpakTv_MIC&rdid=book-ZxwpakTv_MIC&rdot=1&source=gbs_api",
"offers": [
{
"finskyOfferType": 1,
"listPrice": {
"amountInMicros": 3990000,
"currencyCode": "USD"
},
"retailPrice": {
"amountInMicros": 3990000,
"currencyCode": "USD"
},
"giftable": true
}
]
},
"accessInfo": {
"country": "US",
"viewability": "PARTIAL",
"embeddable": true,
"publicDomain": false,
"textToSpeechPermission": "ALLOWED",
"epub": {
"isAvailable": true,
"acsTokenLink": "http://books.google.com/books/download/Alice_in_Wonderland-sample-epub.acsm?id=ZxwpakTv_MIC&format=epub&output=acs4_fulfillment_token&dl_type=sample&source=gbs_api"
},
"pdf": {
"isAvailable": false
},
"webReaderLink": "http://play.google.com/books/reader?id=ZxwpakTv_MIC&hl=&source=gbs_api",
"accessViewStatus": "SAMPLE",
"quoteSharingAllowed": false
},
"searchInfo": {
"textSnippet": "Alice in Wonderland (also known as Alice's Adventures in Wonderland), from 1865, is the peculiar and imaginative tale of a girl who falls down a rabbit-hole into a bizarre world of eccentric and unusual creatures."
}
}
]
}

1
metadata/epub.go
View File

@ -16,6 +16,7 @@ func getEPUBMetadata(filepath string) (*MetadataInfo, error) {
rf := rc.Rootfiles[0]
parsedMetadata := &MetadataInfo{
Type: TYPE_EPUB,
Title: &rf.Title,
Author: &rf.Creator,
Description: &rf.Description,

130
metadata/gbooks_test.go Normal file
View File

@ -0,0 +1,130 @@
package metadata
import (
_ "embed"
"encoding/json"
"fmt"
"net/http"
"net/url"
"regexp"
"strings"
"testing"
"github.com/jarcoal/httpmock"
"github.com/stretchr/testify/assert"
)
// const GBOOKS_QUERY_URL string = "https://www.googleapis.com/books/v1/volumes?q=%s"
// const GBOOKS_GBID_INFO_URL string = "https://www.googleapis.com/books/v1/volumes/%s"
// const GBOOKS_GBID_COVER_URL string = "https://books.google.com/books/content/images/frontcover/%s?fife=w480-h690"
//go:embed _test_files/gbooks_id_response.json
var idResp string
//go:embed _test_files/gbooks_query_response.json
var queryResp string
type details struct {
URLs []string
}
// Hook API Helper
func hookAPI() *details {
// Start HTTPMock
httpmock.Activate()
// Create details struct
d := &details{
URLs: []string{},
}
// Create Hook
matchRE := regexp.MustCompile(`^https://www\.googleapis\.com/books/v1/volumes.*`)
httpmock.RegisterRegexpResponder("GET", matchRE, func(req *http.Request) (*http.Response, error) {
// Append URL
d.URLs = append(d.URLs, req.URL.String())
// Get Raw Response
var rawResp string
if req.URL.Query().Get("q") != "" {
rawResp = queryResp
} else {
rawResp = idResp
}
// Convert to JSON Response
var responseData map[string]interface{}
json.Unmarshal([]byte(rawResp), &responseData)
// Return Response
return httpmock.NewJsonResponse(200, responseData)
})
return d
}
func TestGBooksGBIDMetadata(t *testing.T) {
hookDetails := hookAPI()
defer httpmock.DeactivateAndReset()
GBID := "ZxwpakTv_MIC"
expectedURL := fmt.Sprintf(GBOOKS_GBID_INFO_URL, GBID)
metadataResp, err := getGBooksMetadata(MetadataInfo{ID: &GBID})
assert.Nil(t, err, "should not have error")
assert.Contains(t, hookDetails.URLs, expectedURL, "should have intercepted URL")
assert.Equal(t, 1, len(metadataResp), "should have single result")
mResult := metadataResp[0]
validateResult(t, &mResult)
}
func TestGBooksISBNQuery(t *testing.T) {
hookDetails := hookAPI()
defer httpmock.DeactivateAndReset()
ISBN10 := "1877527815"
expectedURL := fmt.Sprintf(GBOOKS_QUERY_URL, "isbn:"+ISBN10)
metadataResp, err := getGBooksMetadata(MetadataInfo{
ISBN10: &ISBN10,
})
assert.Nil(t, err, "should not have error")
assert.Contains(t, hookDetails.URLs, expectedURL, "should have intercepted URL")
assert.Equal(t, 1, len(metadataResp), "should have single result")
mResult := metadataResp[0]
validateResult(t, &mResult)
}
func TestGBooksTitleQuery(t *testing.T) {
hookDetails := hookAPI()
defer httpmock.DeactivateAndReset()
title := "Alice in Wonderland 1877527815"
expectedURL := fmt.Sprintf(GBOOKS_QUERY_URL, url.QueryEscape(strings.TrimSpace(title)))
metadataResp, err := getGBooksMetadata(MetadataInfo{
Title: &title,
})
assert.Nil(t, err, "should not have error")
assert.Contains(t, hookDetails.URLs, expectedURL, "should have intercepted URL")
assert.NotEqual(t, 0, len(metadataResp), "should not have no results")
mResult := metadataResp[0]
validateResult(t, &mResult)
}
func validateResult(t *testing.T, m *MetadataInfo) {
expectedTitle := "Alice in Wonderland"
expectedAuthor := "Lewis Carroll"
expectedDesc := "Alice in Wonderland (also known as Alice's Adventures in Wonderland), from 1865, is the peculiar and imaginative tale of a girl who falls down a rabbit-hole into a bizarre world of eccentric and unusual creatures. Lewis Carroll's prominent example of the genre of \"literary nonsense\" has endured in popularity with its clever way of playing with logic and a narrative structure that has influence generations of fiction writing."
expectedISBN10 := "1877527815"
expectedISBN13 := "9781877527814"
assert.Equal(t, expectedTitle, *m.Title, "should have title")
assert.Equal(t, expectedAuthor, *m.Author, "should have author")
assert.Equal(t, expectedDesc, *m.Description, "should have description")
assert.Equal(t, expectedISBN10, *m.ISBN10, "should have ISBN10")
assert.Equal(t, expectedISBN13, *m.ISBN13, "should have ISBN10")
}

76
metadata/integrations_test.go
View File

@ -1,76 +0,0 @@
//go:build integration
package metadata
import (
"testing"
)
func TestGBooksGBIDMetadata(t *testing.T) {
GBID := "ZxwpakTv_MIC"
metadataResp, err := getGBooksMetadata(MetadataInfo{
ID: &GBID,
})
if len(metadataResp) != 1 {
t.Fatalf(`Expected: %v, Got: %v, Error: %v`, 1, len(metadataResp), err)
}
mResult := metadataResp[0]
validateResult(&mResult, t)
}
func TestGBooksISBNQuery(t *testing.T) {
ISBN10 := "1877527815"
metadataResp, err := getGBooksMetadata(MetadataInfo{
ISBN10: &ISBN10,
})
if len(metadataResp) != 1 {
t.Fatalf(`Expected: %v, Got: %v, Error: %v`, 1, len(metadataResp), err)
}
mResult := metadataResp[0]
validateResult(&mResult, t)
}
func TestGBooksTitleQuery(t *testing.T) {
title := "Alice in Wonderland 1877527815"
metadataResp, err := getGBooksMetadata(MetadataInfo{
Title: &title,
})
if len(metadataResp) == 0 {
t.Fatalf(`Expected: %v, Got: %v, Error: %v`, "> 0", len(metadataResp), err)
}
mResult := metadataResp[0]
validateResult(&mResult, t)
}
func validateResult(m *MetadataInfo, t *testing.T) {
expect := "Lewis Carroll"
if *m.Author != expect {
t.Fatalf(`Expected: %v, Got: %v`, expect, *m.Author)
}
expect = "Alice in Wonderland"
if *m.Title != expect {
t.Fatalf(`Expected: %v, Got: %v`, expect, *m.Title)
}
expect = "Alice in Wonderland (also known as Alice's Adventures in Wonderland), from 1865, is the peculiar and imaginative tale of a girl who falls down a rabbit-hole into a bizarre world of eccentric and unusual creatures. Lewis Carroll's prominent example of the genre of \"literary nonsense\" has endured in popularity with its clever way of playing with logic and a narrative structure that has influence generations of fiction writing."
if *m.Description != expect {
t.Fatalf(`Expected: %v, Got: %v`, expect, *m.Description)
}
expect = "1877527815"
if *m.ISBN10 != expect {
t.Fatalf(`Expected: %v, Got: %v`, expect, *m.ISBN10)
}
expect = "9781877527814"
if *m.ISBN13 != expect {
t.Fatalf(`Expected: %v, Got: %v`, expect, *m.ISBN13)
}
}

151
metadata/metadata.go
View File

@ -3,27 +3,47 @@ package metadata
import (
"errors"
"fmt"
"io"
"path/filepath"
"github.com/gabriel-vasile/mimetype"
"reichard.io/antholume/utils"
)
type MetadataHandler func(string) (*MetadataInfo, error)
type DocumentType string
const (
TYPE_EPUB DocumentType = ".epub"
)
var extensionHandlerMap = map[DocumentType]MetadataHandler{
TYPE_EPUB: getEPUBMetadata,
}
type Source int
const (
GBOOK Source = iota
OLIB
SOURCE_GBOOK Source = iota
SOURCE_OLIB
)
type MetadataInfo struct {
ID *string
ID *string
MD5 *string
PartialMD5 *string
WordCount *int64
Title *string
Author *string
Description *string
ISBN10 *string
ISBN13 *string
Type DocumentType
}
// Downloads the Google Books cover file and saves it to the provided directory.
func CacheCover(gbid string, coverDir string, documentID string, overwrite bool) (*string, error) {
// Get Filepath
coverFile := "." + filepath.Clean(fmt.Sprintf("/%s.jpg", documentID))
@ -39,11 +59,12 @@ func CacheCover(gbid string, coverDir string, documentID string, overwrite bool)
return &coverFile, nil
}
// Searches source for metadata based on the provided information.
func SearchMetadata(s Source, metadataSearch MetadataInfo) ([]MetadataInfo, error) {
switch s {
case GBOOK:
case SOURCE_GBOOK:
return getGBooksMetadata(metadataSearch)
case OLIB:
case SOURCE_OLIB:
return nil, errors.New("Not implemented")
default:
return nil, errors.New("Not implemented")
@ -51,32 +72,112 @@ func SearchMetadata(s Source, metadataSearch MetadataInfo) ([]MetadataInfo, erro
}
}
func GetWordCount(filepath string) (int64, error) {
fileMime, err := mimetype.DetectFile(filepath)
if err != nil {
return 0, err
}
if fileExtension := fileMime.Extension(); fileExtension == ".epub" {
totalWords, err := countEPUBWords(filepath)
if err != nil {
return 0, err
}
return totalWords, nil
} else {
return 0, errors.New("Invalid Extension")
}
}
func GetMetadata(filepath string) (*MetadataInfo, error) {
// Returns the word count of the provided filepath. An error will be returned
// if the file is not supported.
func GetWordCount(filepath string) (*int64, error) {
fileMime, err := mimetype.DetectFile(filepath)
if err != nil {
return nil, err
}
if fileExtension := fileMime.Extension(); fileExtension == ".epub" {
return getEPUBMetadata(filepath)
totalWords, err := countEPUBWords(filepath)
if err != nil {
return nil, err
}
return &totalWords, nil
} else {
return nil, errors.New("Invalid Extension")
return nil, fmt.Errorf("Invalid extension")
}
}
// Returns embedded metadata of the provided file. An error will be returned if
// the file is not supported.
func GetMetadata(filepath string) (*MetadataInfo, error) {
// Detect Extension Type
fileMime, err := mimetype.DetectFile(filepath)
if err != nil {
return nil, err
}
// Get Extension Type Metadata Handler
fileExtension := fileMime.Extension()
handler, ok := extensionHandlerMap[DocumentType(fileExtension)]
if !ok {
return nil, fmt.Errorf("invalid extension %s", fileExtension)
}
// Acquire Metadata
metadataInfo, err := handler(filepath)
// Calculate MD5 & Partial MD5
partialMD5, err := utils.CalculatePartialMD5(filepath)
if err != nil {
return nil, fmt.Errorf("unable to calculate partial MD5")
}
// Calculate Actual MD5
MD5, err := utils.CalculateMD5(filepath)
if err != nil {
return nil, fmt.Errorf("unable to calculate MD5")
}
// Calculate Word Count
wordCount, err := GetWordCount(filepath)
if err != nil {
return nil, fmt.Errorf("unable to calculate word count")
}
metadataInfo.WordCount = wordCount
metadataInfo.PartialMD5 = partialMD5
metadataInfo.MD5 = MD5
return metadataInfo, nil
}
// Returns the extension of the provided filepath (e.g. ".epub"). An error
// will be returned if the file is not supported.
func GetDocumentType(filepath string) (*DocumentType, error) {
// Detect Extension Type
fileMime, err := mimetype.DetectFile(filepath)
if err != nil {
return nil, err
}
// Detect
fileExtension := fileMime.Extension()
docType, ok := ParseDocumentType(fileExtension)
if !ok {
return nil, fmt.Errorf("filetype not supported")
}
return &docType, nil
}
// Returns the extension of the provided file reader (e.g. ".epub"). An error
// will be returned if the file is not supported.
func GetDocumentTypeReader(r io.Reader) (*DocumentType, error) {
// Detect Extension Type
fileMime, err := mimetype.DetectReader(r)
if err != nil {
return nil, err
}
// Detect
fileExtension := fileMime.Extension()
docType, ok := ParseDocumentType(fileExtension)
if !ok {
return nil, fmt.Errorf("filetype not supported")
}
return &docType, nil
}
// Given a filetype string, attempt to resolve a DocumentType
func ParseDocumentType(input string) (DocumentType, bool) {
validTypes := map[string]DocumentType{
string(TYPE_EPUB): TYPE_EPUB,
}
found, ok := validTypes[input]
return found, ok
}

54
metadata/metadata_test.go
View File

@ -1,36 +1,46 @@
package metadata
import (
"os"
"testing"
"github.com/stretchr/testify/assert"
)
func TestGetWordCount(t *testing.T) {
var want int64 = 30080
wordCount, err := countEPUBWords("../_test_files/alice.epub")
var desiredCount int64 = 30080
actualCount, err := countEPUBWords("../_test_files/alice.epub")
assert.Nil(t, err, "should have no error")
assert.Equal(t, desiredCount, actualCount, "should be correct word count")
if wordCount != want {
t.Fatalf(`Expected: %v, Got: %v, Error: %v`, want, wordCount, err)
}
}
func TestGetMetadata(t *testing.T) {
metadataInfo, err := getEPUBMetadata("../_test_files/alice.epub")
if err != nil {
t.Fatalf(`Expected: *MetadataInfo, Got: nil, Error: %v`, err)
}
desiredTitle := "Alice's Adventures in Wonderland / Illustrated by Arthur Rackham. With a Proem by Austin Dobson"
desiredAuthor := "Lewis Carroll"
desiredDescription := ""
want := "Alice's Adventures in Wonderland / Illustrated by Arthur Rackham. With a Proem by Austin Dobson"
if *metadataInfo.Title != want {
t.Fatalf(`Expected: %v, Got: %v, Error: %v`, want, *metadataInfo.Title, err)
}
metadataInfo, err := GetMetadata("../_test_files/alice.epub")
want = "Lewis Carroll"
if *metadataInfo.Author != want {
t.Fatalf(`Expected: %v, Got: %v, Error: %v`, want, *metadataInfo.Author, err)
}
want = ""
if *metadataInfo.Description != want {
t.Fatalf(`Expected: %v, Got: %v, Error: %v`, want, *metadataInfo.Description, err)
}
assert.Nil(t, err, "should have no error")
assert.Equal(t, desiredTitle, *metadataInfo.Title, "should be correct title")
assert.Equal(t, desiredAuthor, *metadataInfo.Author, "should be correct author")
assert.Equal(t, desiredDescription, *metadataInfo.Description, "should be correct author")
assert.Equal(t, TYPE_EPUB, metadataInfo.Type, "should be correct type")
}
func TestGetExtension(t *testing.T) {
docType, err := GetDocumentType("../_test_files/alice.epub")
assert.Nil(t, err, "should have no error")
assert.Equal(t, TYPE_EPUB, *docType)
}
func TestGetExtensionReader(t *testing.T) {
file, _ := os.Open("../_test_files/alice.epub")
docType, err := GetDocumentTypeReader(file)
assert.Nil(t, err, "should have no error")
assert.Equal(t, TYPE_EPUB, *docType)
}

2
opds/opds.go
View File

@ -8,8 +8,8 @@ import (
// Feed root element for acquisition or navigation feed
type Feed struct {
ID string `xml:"id,omitempty"`
XMLName xml.Name `xml:"feed"`
ID string `xml:"id,omitempty",`
Title string `xml:"title,omitempty"`
Updated time.Time `xml:"updated,omitempty"`
Entries []Entry `xml:"entry,omitempty"`

75
search/anna.go Normal file
View File

@ -0,0 +1,75 @@
package search
import (
"fmt"
"io"
"strings"
"github.com/PuerkitoBio/goquery"
)
func parseAnnasArchiveDownloadURL(body io.ReadCloser) (string, error) {
// Parse
defer body.Close()
doc, _ := goquery.NewDocumentFromReader(body)
// Return Download URL
downloadURL, exists := doc.Find("body > table > tbody > tr > td > a").Attr("href")
if exists == false {
return "", fmt.Errorf("Download URL not found")
}
// Possible Funky URL
downloadURL = strings.ReplaceAll(downloadURL, "\\", "/")
return downloadURL, nil
}
func parseAnnasArchive(body io.ReadCloser) ([]SearchItem, error) {
// Parse
defer body.Close()
doc, err := goquery.NewDocumentFromReader(body)
if err != nil {
return nil, err
}
// Normalize Results
var allEntries []SearchItem
doc.Find("form > div.w-full > div.w-full > div > div.justify-center").Each(func(ix int, rawBook *goquery.Selection) {
// Parse Details
details := rawBook.Find("div:nth-child(2) > div:nth-child(1)").Text()
detailsSplit := strings.Split(details, ", ")
// Invalid Details
if len(detailsSplit) < 3 {
return
}
language := detailsSplit[0]
fileType := detailsSplit[1]
fileSize := detailsSplit[2]
// Get Title & Author
title := rawBook.Find("h3").Text()
author := rawBook.Find("div:nth-child(2) > div:nth-child(4)").Text()
// Parse MD5
itemHref, _ := rawBook.Find("a").Attr("href")
hrefArray := strings.Split(itemHref, "/")
id := hrefArray[len(hrefArray)-1]
item := SearchItem{
ID: id,
Title: title,
Author: author,
Language: language,
FileType: fileType,
FileSize: fileSize,
}
allEntries = append(allEntries, item)
})
// Return Results
return allEntries, nil
}

42
search/goodreads.go Normal file
View File

@ -0,0 +1,42 @@
package search
import (
"io"
"github.com/PuerkitoBio/goquery"
)
func GoodReadsMostRead(c Cadence) ([]SearchItem, error) {
body, err := getPage("https://www.goodreads.com/book/most_read?category=all&country=US&duration=" + string(c))
if err != nil {
return nil, err
}
return parseGoodReads(body)
}
func parseGoodReads(body io.ReadCloser) ([]SearchItem, error) {
// Parse
defer body.Close()
doc, err := goquery.NewDocumentFromReader(body)
if err != nil {
return nil, err
}
// Normalize Results
var allEntries []SearchItem
doc.Find("[itemtype=\"http://schema.org/Book\"]").Each(func(ix int, rawBook *goquery.Selection) {
title := rawBook.Find(".bookTitle span").Text()
author := rawBook.Find(".authorName span").Text()
item := SearchItem{
Title: title,
Author: author,
}
allEntries = append(allEntries, item)
})
// Return Results
return allEntries, nil
}

123
search/libgen.go Normal file
View File

@ -0,0 +1,123 @@
package search
import (
"fmt"
"io"
"strings"
"time"
"github.com/PuerkitoBio/goquery"
)
func parseLibGenFiction(body io.ReadCloser) ([]SearchItem, error) {
// Parse
defer body.Close()
doc, err := goquery.NewDocumentFromReader(body)
if err != nil {
return nil, err
}
// Normalize Results
var allEntries []SearchItem
doc.Find("table.catalog tbody > tr").Each(func(ix int, rawBook *goquery.Selection) {
// Parse File Details
fileItem := rawBook.Find("td:nth-child(5)")
fileDesc := fileItem.Text()
fileDescSplit := strings.Split(fileDesc, "/")
fileType := strings.ToLower(strings.TrimSpace(fileDescSplit[0]))
fileSize := strings.TrimSpace(fileDescSplit[1])
// Parse Upload Date
uploadedRaw, _ := fileItem.Attr("title")
uploadedDateRaw := strings.Split(uploadedRaw, "Uploaded at ")[1]
uploadDate, _ := time.Parse("2006-01-02 15:04:05", uploadedDateRaw)
// Parse MD5
editHref, _ := rawBook.Find("td:nth-child(7) a").Attr("href")
hrefArray := strings.Split(editHref, "/")
id := hrefArray[len(hrefArray)-1]
// Parse Other Details
title := rawBook.Find("td:nth-child(3) p a").Text()
author := rawBook.Find(".catalog_authors li a").Text()
language := rawBook.Find("td:nth-child(4)").Text()
series := rawBook.Find("td:nth-child(2)").Text()
item := SearchItem{
ID: id,
Title: title,
Author: author,
Series: series,
Language: language,
FileType: fileType,
FileSize: fileSize,
UploadDate: uploadDate.Format(time.RFC3339),
}
allEntries = append(allEntries, item)
})
// Return Results
return allEntries, nil
}
func parseLibGenNonFiction(body io.ReadCloser) ([]SearchItem, error) {
// Parse
defer body.Close()
doc, err := goquery.NewDocumentFromReader(body)
if err != nil {
return nil, err
}
// Normalize Results
var allEntries []SearchItem
doc.Find("table.c tbody > tr:nth-child(n + 2)").Each(func(ix int, rawBook *goquery.Selection) {
// Parse Type & Size
fileSize := strings.ToLower(strings.TrimSpace(rawBook.Find("td:nth-child(8)").Text()))
fileType := strings.ToLower(strings.TrimSpace(rawBook.Find("td:nth-child(9)").Text()))
// Parse MD5
titleRaw := rawBook.Find("td:nth-child(3) [id]")
editHref, _ := titleRaw.Attr("href")
hrefArray := strings.Split(editHref, "?md5=")
id := hrefArray[1]
// Parse Other Details
title := titleRaw.Text()
author := rawBook.Find("td:nth-child(2)").Text()
language := rawBook.Find("td:nth-child(7)").Text()
series := rawBook.Find("td:nth-child(3) [href*='column=series']").Text()
item := SearchItem{
ID: id,
Title: title,
Author: author,
Series: series,
Language: language,
FileType: fileType,
FileSize: fileSize,
}
allEntries = append(allEntries, item)
})
// Return Results
return allEntries, nil
}
func parseLibGenDownloadURL(body io.ReadCloser) (string, error) {
// Parse
defer body.Close()
doc, _ := goquery.NewDocumentFromReader(body)
// Return Download URL
// downloadURL, _ := doc.Find("#download [href*=cloudflare]").Attr("href")
downloadURL, exists := doc.Find("#download h2 a").Attr("href")
if exists == false {
return "", fmt.Errorf("Download URL not found")
}
return downloadURL, nil
}

225
search/search.go
View File

@ -2,16 +2,13 @@ package search
import (
"crypto/tls"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"os"
"strings"
"time"
"github.com/PuerkitoBio/goquery"
log "github.com/sirupsen/logrus"
)
@ -102,14 +99,14 @@ func SaveBook(id string, source Source) (string, error) {
bookURL, err := def.parseDownloadFunc(body)
if err != nil {
log.Error("Parse Download URL Error: ", err)
return "", errors.New("Download Failure")
return "", fmt.Errorf("Download Failure")
}
// Create File
tempFile, err := os.CreateTemp("", "book")
if err != nil {
log.Error("File Create Error: ", err)
return "", errors.New("File Failure")
return "", fmt.Errorf("File Failure")
}
defer tempFile.Close()
@ -119,7 +116,7 @@ func SaveBook(id string, source Source) (string, error) {
if err != nil {
os.Remove(tempFile.Name())
log.Error("Book URL API Failure: ", err)
return "", errors.New("API Failure")
return "", fmt.Errorf("API Failure")
}
defer resp.Body.Close()
@ -129,20 +126,12 @@ func SaveBook(id string, source Source) (string, error) {
if err != nil {
os.Remove(tempFile.Name())
log.Error("File Copy Error: ", err)
return "", errors.New("File Failure")
return "", fmt.Errorf("File Failure")
}
return tempFile.Name(), nil
}
func GoodReadsMostRead(c Cadence) ([]SearchItem, error) {
body, err := getPage("https://www.goodreads.com/book/most_read?category=all&country=US&duration=" + string(c))
if err != nil {
return nil, err
}
return parseGoodReads(body)
}
func GetBookURL(id string, bookType BookType) (string, error) {
// Derive Info URL
var infoURL string
@ -180,212 +169,6 @@ func getPage(page string) (io.ReadCloser, error) {
return resp.Body, err
}
func parseLibGenFiction(body io.ReadCloser) ([]SearchItem, error) {
// Parse
defer body.Close()
doc, err := goquery.NewDocumentFromReader(body)
if err != nil {
return nil, err
}
// Normalize Results
var allEntries []SearchItem
doc.Find("table.catalog tbody > tr").Each(func(ix int, rawBook *goquery.Selection) {
// Parse File Details
fileItem := rawBook.Find("td:nth-child(5)")
fileDesc := fileItem.Text()
fileDescSplit := strings.Split(fileDesc, "/")
fileType := strings.ToLower(strings.TrimSpace(fileDescSplit[0]))
fileSize := strings.TrimSpace(fileDescSplit[1])
// Parse Upload Date
uploadedRaw, _ := fileItem.Attr("title")
uploadedDateRaw := strings.Split(uploadedRaw, "Uploaded at ")[1]
uploadDate, _ := time.Parse("2006-01-02 15:04:05", uploadedDateRaw)
// Parse MD5
editHref, _ := rawBook.Find("td:nth-child(7) a").Attr("href")
hrefArray := strings.Split(editHref, "/")
id := hrefArray[len(hrefArray)-1]
// Parse Other Details
title := rawBook.Find("td:nth-child(3) p a").Text()
author := rawBook.Find(".catalog_authors li a").Text()
language := rawBook.Find("td:nth-child(4)").Text()
series := rawBook.Find("td:nth-child(2)").Text()
item := SearchItem{
ID: id,
Title: title,
Author: author,
Series: series,
Language: language,
FileType: fileType,
FileSize: fileSize,
UploadDate: uploadDate.Format(time.RFC3339),
}
allEntries = append(allEntries, item)
})
// Return Results
return allEntries, nil
}
func parseLibGenNonFiction(body io.ReadCloser) ([]SearchItem, error) {
// Parse
defer body.Close()
doc, err := goquery.NewDocumentFromReader(body)
if err != nil {
return nil, err
}
// Normalize Results
var allEntries []SearchItem
doc.Find("table.c tbody > tr:nth-child(n + 2)").Each(func(ix int, rawBook *goquery.Selection) {
// Parse Type & Size
fileSize := strings.ToLower(strings.TrimSpace(rawBook.Find("td:nth-child(8)").Text()))
fileType := strings.ToLower(strings.TrimSpace(rawBook.Find("td:nth-child(9)").Text()))
// Parse MD5
titleRaw := rawBook.Find("td:nth-child(3) [id]")
editHref, _ := titleRaw.Attr("href")
hrefArray := strings.Split(editHref, "?md5=")
id := hrefArray[1]
// Parse Other Details
title := titleRaw.Text()
author := rawBook.Find("td:nth-child(2)").Text()
language := rawBook.Find("td:nth-child(7)").Text()
series := rawBook.Find("td:nth-child(3) [href*='column=series']").Text()
item := SearchItem{
ID: id,
Title: title,
Author: author,
Series: series,
Language: language,
FileType: fileType,
FileSize: fileSize,
}
allEntries = append(allEntries, item)
})
// Return Results
return allEntries, nil
}
func parseLibGenDownloadURL(body io.ReadCloser) (string, error) {
// Parse
defer body.Close()
doc, _ := goquery.NewDocumentFromReader(body)
// Return Download URL
// downloadURL, _ := doc.Find("#download [href*=cloudflare]").Attr("href")
downloadURL, exists := doc.Find("#download h2 a").Attr("href")
if exists == false {
return "", errors.New("Download URL not found")
}
return downloadURL, nil
}
func parseGoodReads(body io.ReadCloser) ([]SearchItem, error) {
// Parse
defer body.Close()
doc, err := goquery.NewDocumentFromReader(body)
if err != nil {
return nil, err
}
// Normalize Results
var allEntries []SearchItem
doc.Find("[itemtype=\"http://schema.org/Book\"]").Each(func(ix int, rawBook *goquery.Selection) {
title := rawBook.Find(".bookTitle span").Text()
author := rawBook.Find(".authorName span").Text()
item := SearchItem{
Title: title,
Author: author,
}
allEntries = append(allEntries, item)
})
// Return Results
return allEntries, nil
}
func parseAnnasArchiveDownloadURL(body io.ReadCloser) (string, error) {
// Parse
defer body.Close()
doc, _ := goquery.NewDocumentFromReader(body)
// Return Download URL
downloadURL, exists := doc.Find("body > table > tbody > tr > td > a").Attr("href")
if exists == false {
return "", errors.New("Download URL not found")
}
// Possible Funky URL
downloadURL = strings.ReplaceAll(downloadURL, "\\", "/")
return downloadURL, nil
}
func parseAnnasArchive(body io.ReadCloser) ([]SearchItem, error) {
// Parse
defer body.Close()
doc, err := goquery.NewDocumentFromReader(body)
if err != nil {
return nil, err
}
// Normalize Results
var allEntries []SearchItem
doc.Find("form > div.w-full > div.w-full > div > div.justify-center").Each(func(ix int, rawBook *goquery.Selection) {
// Parse Details
details := rawBook.Find("div:nth-child(2) > div:nth-child(1)").Text()
detailsSplit := strings.Split(details, ", ")
// Invalid Details
if len(detailsSplit) < 3 {
return
}
language := detailsSplit[0]
fileType := detailsSplit[1]
fileSize := detailsSplit[2]
// Get Title & Author
title := rawBook.Find("h3").Text()
author := rawBook.Find("div:nth-child(2) > div:nth-child(4)").Text()
// Parse MD5
itemHref, _ := rawBook.Find("a").Attr("href")
hrefArray := strings.Split(itemHref, "/")
id := hrefArray[len(hrefArray)-1]
item := SearchItem{
ID: id,
Title: title,
Author: author,
Language: language,
FileType: fileType,
FileSize: fileSize,
}
allEntries = append(allEntries, item)
})
// Return Results
return allEntries, nil
}
func downloadBook(bookURL string) (*http.Response, error) {
// Allow Insecure
client := &http.Client{Transport: &http.Transport{

27
utils/utils.go
View File

@ -10,10 +10,10 @@ import (
)
// Reimplemented KOReader Partial MD5 Calculation
func CalculatePartialMD5(filePath string) (string, error) {
func CalculatePartialMD5(filePath string) (*string, error) {
file, err := os.Open(filePath)
if err != nil {
return "", err
return nil, err
}
defer file.Close()
@ -41,7 +41,8 @@ func CalculatePartialMD5(filePath string) (string, error) {
}
allBytes := buf.Bytes()
return fmt.Sprintf("%x", md5.Sum(allBytes)), nil
fileHash := fmt.Sprintf("%x", md5.Sum(allBytes))
return &fileHash, nil
}
// Creates a token of n size
@ -53,3 +54,23 @@ func GenerateToken(n int) ([]byte, error) {
}
return b, nil
}
// Calculate MD5 of a file
func CalculateMD5(filePath string) (*string, error) {
file, err := os.Open(filePath)
if err != nil {
return nil, err
}
defer file.Close()
hash := md5.New()
_, err = io.Copy(hash, file)
if err != nil {
return nil, err
}
fileHash := fmt.Sprintf("%x", hash.Sum(nil))
return &fileHash, nil
}

26
utils/utils_test.go
View File

@ -1,12 +1,26 @@
package utils
import "testing"
import (
"github.com/stretchr/testify/assert"
"testing"
)
func TestCalculatePartialMD5(t *testing.T) {
partialMD5, err := CalculatePartialMD5("../_test_files/alice.epub")
assert := assert.New(t)
want := "386d1cb51fe4a72e5c9fdad5e059bad9"
if partialMD5 != want {
t.Fatalf(`Expected: %v, Got: %v, Error: %v`, want, partialMD5, err)
}
desiredPartialMD5 := "386d1cb51fe4a72e5c9fdad5e059bad9"
calculatedPartialMD5, err := CalculatePartialMD5("../_test_files/alice.epub")
assert.Nil(err, "error should be nil")
assert.Equal(desiredPartialMD5, *calculatedPartialMD5, "should be equal")
}
func TestCalculateMD5(t *testing.T) {
assert := assert.New(t)
desiredMD5 := "0f36c66155de34b281c4791654d0b1ce"
calculatedMD5, err := CalculateMD5("../_test_files/alice.epub")
assert.Nil(err, "error should be nil")
assert.Equal(desiredMD5, *calculatedMD5, "should be equal")
}