Compare commits

...

3 Commits

Author SHA1 Message Date
75ed394f8d tests(all): improve tests, refactor(api): saving books
All checks were successful
continuous-integration/drone/push Build is passing
2024-02-24 20:45:26 -05:00
803c187a00 fix(logs): ios pretty logs & overflow scroll 2024-02-24 17:07:12 -05:00
da1baeb4cd feat(reader): upgrade epubjs & add restrictive iframe CSP 2024-02-19 16:45:35 -05:00
32 changed files with 1136 additions and 697 deletions

View File

@ -4,21 +4,10 @@ name: default
steps:
# Unit Tests
- name: unit test
- name: tests
image: golang
commands:
- make tests_unit
# Integration Tests (Every Month)
- name: integration test
image: golang
commands:
- make tests_integration
when:
event:
- cron
cron:
- integration-test
- make tests
# Fetch tags
- name: fetch tags

1
.gitignore vendored
View File

@ -3,3 +3,4 @@ TODO.md
data/
build/
.direnv/
cover.html

View File

@ -42,8 +42,7 @@ dev: build_tailwind
clean:
rm -rf ./build
tests_integration:
go test -v -tags=integration -coverpkg=./... ./metadata
tests_unit:
SET_TEST=set_val go test -v -coverpkg=./... ./...
tests:
SET_TEST=set_val go test -coverpkg=./... ./... -coverprofile=./cover.out
go tool cover -html=./cover.out -o ./cover.html
rm ./cover.out

View File

@ -19,6 +19,7 @@ import (
"github.com/gin-gonic/gin"
"github.com/itchyny/gojq"
log "github.com/sirupsen/logrus"
"reichard.io/antholume/metadata"
)
type adminAction string
@ -63,7 +64,7 @@ func (api *API) appPerformAdminAction(c *gin.Context) {
var rAdminAction requestAdminAction
if err := c.ShouldBind(&rAdminAction); err != nil {
log.Error("Invalid Form Bind: ", err)
appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values.")
appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values")
return
}
@ -75,6 +76,7 @@ func (api *API) appPerformAdminAction(c *gin.Context) {
// 2. Select all / deselect?
case adminCacheTables:
go api.db.CacheTempTables()
// TODO - Message
case adminRestore:
api.processRestoreFile(rAdminAction, c)
return
@ -83,7 +85,7 @@ func (api *API) appPerformAdminAction(c *gin.Context) {
_, err := api.db.DB.ExecContext(api.db.Ctx, "VACUUM;")
if err != nil {
log.Error("Unable to vacuum DB: ", err)
appErrorPage(c, http.StatusInternalServerError, "Unable to vacuum database.")
appErrorPage(c, http.StatusInternalServerError, "Unable to vacuum database")
return
}
@ -126,7 +128,7 @@ func (api *API) appGetAdminLogs(c *gin.Context) {
var rAdminLogs requestAdminLogs
if err := c.ShouldBindQuery(&rAdminLogs); err != nil {
log.Error("Invalid URI Bind")
appErrorPage(c, http.StatusNotFound, "Invalid URI parameters.")
appErrorPage(c, http.StatusNotFound, "Invalid URI parameters")
return
}
rAdminLogs.Filter = strings.TrimSpace(rAdminLogs.Filter)
@ -136,14 +138,14 @@ func (api *API) appGetAdminLogs(c *gin.Context) {
parsed, err := gojq.Parse(rAdminLogs.Filter)
if err != nil {
log.Error("Unable to parse JQ filter")
appErrorPage(c, http.StatusNotFound, "Unable to parse JQ filter.")
appErrorPage(c, http.StatusNotFound, "Unable to parse JQ filter")
return
}
jqFilter, err = gojq.Compile(parsed)
if err != nil {
log.Error("Unable to compile JQ filter")
appErrorPage(c, http.StatusNotFound, "Unable to compile JQ filter.")
appErrorPage(c, http.StatusNotFound, "Unable to compile JQ filter")
return
}
}
@ -152,7 +154,7 @@ func (api *API) appGetAdminLogs(c *gin.Context) {
logPath := filepath.Join(api.cfg.ConfigPath, "logs/antholume.log")
logFile, err := os.Open(logPath)
if err != nil {
appErrorPage(c, http.StatusBadRequest, "Missing AnthoLume log file.")
appErrorPage(c, http.StatusBadRequest, "Missing AnthoLume log file")
return
}
defer logFile.Close()
@ -229,7 +231,7 @@ func (api *API) appGetAdminImport(c *gin.Context) {
var rImportFolder requestAdminImport
if err := c.ShouldBindQuery(&rImportFolder); err != nil {
log.Error("Invalid URI Bind")
appErrorPage(c, http.StatusNotFound, "Invalid directory.")
appErrorPage(c, http.StatusNotFound, "Invalid directory")
return
}
@ -244,7 +246,7 @@ func (api *API) appGetAdminImport(c *gin.Context) {
dPath, err := filepath.Abs(api.cfg.DataPath)
if err != nil {
log.Error("Absolute filepath error: ", rImportFolder.Directory)
appErrorPage(c, http.StatusNotFound, "Unable to get data directory absolute path.")
appErrorPage(c, http.StatusNotFound, "Unable to get data directory absolute path")
return
}
@ -254,7 +256,7 @@ func (api *API) appGetAdminImport(c *gin.Context) {
entries, err := os.ReadDir(rImportFolder.Directory)
if err != nil {
log.Error("Invalid directory: ", rImportFolder.Directory)
appErrorPage(c, http.StatusNotFound, "Invalid directory.")
appErrorPage(c, http.StatusNotFound, "Invalid directory")
return
}
@ -279,13 +281,46 @@ func (api *API) appPerformAdminImport(c *gin.Context) {
var rAdminImport requestAdminImport
if err := c.ShouldBind(&rAdminImport); err != nil {
log.Error("Invalid URI Bind")
appErrorPage(c, http.StatusNotFound, "Invalid directory.")
appErrorPage(c, http.StatusNotFound, "Invalid directory")
return
}
// TODO
// TODO - Store results for approval?
fmt.Println(rAdminImport)
// Walk import directory & copy or import files
importDirectory := filepath.Clean(rAdminImport.Directory)
_ = filepath.WalkDir(importDirectory, func(currentPath string, f fs.DirEntry, err error) error {
if err != nil {
return err
}
if f.IsDir() {
return nil
}
// Get metadata
fileMeta, err := metadata.GetMetadata(currentPath)
if err != nil {
fmt.Printf("metadata error: %v\n", err)
return nil
}
// Only needed if copying
newName := deriveBaseFileName(fileMeta)
// Open File on Disk
// file, err := os.Open(currentPath)
// if err != nil {
// return err
// }
// defer file.Close()
// TODO - BasePath in DB
// TODO - Copy / Import
fmt.Printf("New File Metadata: %s\n", newName)
return nil
})
templateVars["CurrentPath"] = filepath.Clean(rAdminImport.Directory)
@ -297,14 +332,14 @@ func (api *API) processRestoreFile(rAdminAction requestAdminAction, c *gin.Conte
uploadedFile, err := rAdminAction.RestoreFile.Open()
if err != nil {
log.Error("File Error: ", err)
appErrorPage(c, http.StatusInternalServerError, "Unable to open file.")
appErrorPage(c, http.StatusInternalServerError, "Unable to open file")
return
}
fileMime, err := mimetype.DetectReader(uploadedFile)
if err != nil {
log.Error("MIME Error")
appErrorPage(c, http.StatusInternalServerError, "Unable to detect filetype.")
appErrorPage(c, http.StatusInternalServerError, "Unable to detect filetype")
return
}
fileExtension := fileMime.Extension()
@ -312,7 +347,7 @@ func (api *API) processRestoreFile(rAdminAction requestAdminAction, c *gin.Conte
// Validate Extension
if !slices.Contains([]string{".zip"}, fileExtension) {
log.Error("Invalid FileType: ", fileExtension)
appErrorPage(c, http.StatusBadRequest, "Invalid filetype.")
appErrorPage(c, http.StatusBadRequest, "Invalid filetype")
return
}
@ -320,7 +355,7 @@ func (api *API) processRestoreFile(rAdminAction requestAdminAction, c *gin.Conte
tempFile, err := os.CreateTemp("", "restore")
if err != nil {
log.Warn("Temp File Create Error: ", err)
appErrorPage(c, http.StatusInternalServerError, "Unable to create temp file.")
appErrorPage(c, http.StatusInternalServerError, "Unable to create temp file")
return
}
defer os.Remove(tempFile.Name())
@ -330,7 +365,7 @@ func (api *API) processRestoreFile(rAdminAction requestAdminAction, c *gin.Conte
err = c.SaveUploadedFile(rAdminAction.RestoreFile, tempFile.Name())
if err != nil {
log.Error("File Error: ", err)
appErrorPage(c, http.StatusInternalServerError, "Unable to save file.")
appErrorPage(c, http.StatusInternalServerError, "Unable to save file")
return
}
@ -338,7 +373,7 @@ func (api *API) processRestoreFile(rAdminAction requestAdminAction, c *gin.Conte
fileInfo, err := tempFile.Stat()
if err != nil {
log.Error("File Error: ", err)
appErrorPage(c, http.StatusInternalServerError, "Unable to read file.")
appErrorPage(c, http.StatusInternalServerError, "Unable to read file")
return
}
@ -346,7 +381,7 @@ func (api *API) processRestoreFile(rAdminAction requestAdminAction, c *gin.Conte
zipReader, err := zip.NewReader(tempFile, fileInfo.Size())
if err != nil {
log.Error("ZIP Error: ", err)
appErrorPage(c, http.StatusInternalServerError, "Unable to read zip.")
appErrorPage(c, http.StatusInternalServerError, "Unable to read zip")
return
}
@ -380,7 +415,7 @@ func (api *API) processRestoreFile(rAdminAction requestAdminAction, c *gin.Conte
backupFile, err := os.Create(backupFilePath)
if err != nil {
log.Error("Unable to create backup file: ", err)
appErrorPage(c, http.StatusInternalServerError, "Unable to create backup file.")
appErrorPage(c, http.StatusInternalServerError, "Unable to create backup file")
return
}
defer backupFile.Close()
@ -389,7 +424,7 @@ func (api *API) processRestoreFile(rAdminAction requestAdminAction, c *gin.Conte
_, err = api.db.DB.ExecContext(api.db.Ctx, "VACUUM;")
if err != nil {
log.Error("Unable to vacuum DB: ", err)
appErrorPage(c, http.StatusInternalServerError, "Unable to vacuum database.")
appErrorPage(c, http.StatusInternalServerError, "Unable to vacuum database")
return
}
@ -398,7 +433,7 @@ func (api *API) processRestoreFile(rAdminAction requestAdminAction, c *gin.Conte
err = api.createBackup(w, []string{"covers", "documents"})
if err != nil {
log.Error("Unable to save backup file: ", err)
appErrorPage(c, http.StatusInternalServerError, "Unable to save backup file.")
appErrorPage(c, http.StatusInternalServerError, "Unable to save backup file")
return
}
@ -406,26 +441,26 @@ func (api *API) processRestoreFile(rAdminAction requestAdminAction, c *gin.Conte
err = api.removeData()
if err != nil {
log.Error("Unable to delete data: ", err)
appErrorPage(c, http.StatusInternalServerError, "Unable to delete data.")
appErrorPage(c, http.StatusInternalServerError, "Unable to delete data")
return
}
// Restore Data
err = api.restoreData(zipReader)
if err != nil {
appErrorPage(c, http.StatusInternalServerError, "Unable to restore data.")
appErrorPage(c, http.StatusInternalServerError, "Unable to restore data")
log.Panic("Unable to restore data: ", err)
}
// Reinit DB
if err := api.db.Reload(); err != nil {
appErrorPage(c, http.StatusInternalServerError, "Unable to reload DB.")
appErrorPage(c, http.StatusInternalServerError, "Unable to reload DB")
log.Panicf("Unable to reload DB: %v", err)
}
// Rotate Auth Hashes
if err := api.rotateAllAuthHashes(); err != nil {
appErrorPage(c, http.StatusInternalServerError, "Unable to rotate hashes.")
appErrorPage(c, http.StatusInternalServerError, "Unable to rotate hashes")
log.Panicf("Unable to rotate auth hashes: %v", err)
}
@ -433,6 +468,7 @@ func (api *API) processRestoreFile(rAdminAction requestAdminAction, c *gin.Conte
c.Redirect(http.StatusFound, "/login")
}
// Restore all data
func (api *API) restoreData(zipReader *zip.Reader) error {
// Ensure Directories
api.cfg.EnsureDirectories()
@ -463,6 +499,7 @@ func (api *API) restoreData(zipReader *zip.Reader) error {
return nil
}
// Remove all data
func (api *API) removeData() error {
allPaths := []string{
"covers",
@ -485,6 +522,7 @@ func (api *API) removeData() error {
return nil
}
// Backup all data
func (api *API) createBackup(w io.Writer, directories []string) error {
ar := zip.NewWriter(w)

View File

@ -157,7 +157,7 @@ func (api *API) appGetDocument(c *gin.Context) {
var rDocID requestDocumentID
if err := c.ShouldBindUri(&rDocID); err != nil {
log.Error("Invalid URI Bind")
appErrorPage(c, http.StatusNotFound, "Invalid document.")
appErrorPage(c, http.StatusNotFound, "Invalid document")
return
}
@ -361,7 +361,7 @@ func (api *API) appGetDocumentProgress(c *gin.Context) {
var rDoc requestDocumentID
if err := c.ShouldBindUri(&rDoc); err != nil {
log.Error("Invalid URI Bind")
appErrorPage(c, http.StatusNotFound, "Invalid document.")
appErrorPage(c, http.StatusNotFound, "Invalid document")
return
}
@ -417,7 +417,7 @@ func (api *API) appUploadNewDocument(c *gin.Context) {
var rDocUpload requestDocumentUpload
if err := c.ShouldBind(&rDocUpload); err != nil {
log.Error("Invalid Form Bind")
appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values.")
appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values")
return
}
@ -426,153 +426,92 @@ func (api *API) appUploadNewDocument(c *gin.Context) {
return
}
// Validate Type & Derive Extension on MIME
uploadedFile, err := rDocUpload.DocumentFile.Open()
if err != nil {
log.Error("File Error: ", err)
appErrorPage(c, http.StatusInternalServerError, "Unable to open file.")
return
}
fileMime, err := mimetype.DetectReader(uploadedFile)
if err != nil {
log.Error("MIME Error")
appErrorPage(c, http.StatusInternalServerError, "Unable to detect filetype.")
return
}
fileExtension := fileMime.Extension()
// Validate Extension
if !slices.Contains([]string{".epub"}, fileExtension) {
log.Error("Invalid FileType: ", fileExtension)
appErrorPage(c, http.StatusBadRequest, "Invalid filetype.")
return
}
// Create Temp File
tempFile, err := os.CreateTemp("", "book")
if err != nil {
log.Warn("Temp File Create Error: ", err)
appErrorPage(c, http.StatusInternalServerError, "Unable to create temp file.")
appErrorPage(c, http.StatusInternalServerError, "Unable to create temp file")
return
}
defer os.Remove(tempFile.Name())
defer tempFile.Close()
// Save Temp
// Save Temp File
err = c.SaveUploadedFile(rDocUpload.DocumentFile, tempFile.Name())
if err != nil {
log.Error("File Error: ", err)
appErrorPage(c, http.StatusInternalServerError, "Unable to save file.")
appErrorPage(c, http.StatusInternalServerError, "Unable to save file")
return
}
// Get Metadata
metadataInfo, err := metadata.GetMetadata(tempFile.Name())
if err != nil {
log.Warn("GetMetadata Error: ", err)
appErrorPage(c, http.StatusInternalServerError, "Unable to acquire file metadata.")
log.Errorf("unable to acquire metadata: %v", err)
appErrorPage(c, http.StatusInternalServerError, "Unable to acquire metadata")
return
}
// Calculate Partial MD5 ID
partialMD5, err := utils.CalculatePartialMD5(tempFile.Name())
if err != nil {
log.Warn("Partial MD5 Error: ", err)
appErrorPage(c, http.StatusInternalServerError, "Unable to calculate partial MD5.")
return
}
// Check Exists
_, err = api.db.Queries.GetDocument(api.db.Ctx, partialMD5)
// Check Already Exists
_, err = api.db.Queries.GetDocument(api.db.Ctx, *metadataInfo.PartialMD5)
if err == nil {
c.Redirect(http.StatusFound, fmt.Sprintf("./documents/%s", partialMD5))
return
log.Warnf("document already exists: %s", *metadataInfo.PartialMD5)
c.Redirect(http.StatusFound, fmt.Sprintf("./documents/%s", *metadataInfo.PartialMD5))
}
// Calculate Actual MD5
fileHash, err := getFileMD5(tempFile.Name())
if err != nil {
log.Error("MD5 Hash Failure: ", err)
appErrorPage(c, http.StatusInternalServerError, "Unable to calculate MD5.")
return
}
// Get Word Count
wordCount, err := metadata.GetWordCount(tempFile.Name())
if err != nil {
log.Error("Word Count Failure: ", err)
appErrorPage(c, http.StatusInternalServerError, "Unable to calculate word count.")
return
}
// Derive Filename
var fileName string
if *metadataInfo.Author != "" {
fileName = fileName + *metadataInfo.Author
} else {
fileName = fileName + "Unknown"
}
if *metadataInfo.Title != "" {
fileName = fileName + " - " + *metadataInfo.Title
} else {
fileName = fileName + " - Unknown"
}
// Remove Slashes
fileName = strings.ReplaceAll(fileName, "/", "")
// Derive & Sanitize File Name
fileName = "." + filepath.Clean(fmt.Sprintf("/%s [%s]%s", fileName, partialMD5, fileExtension))
// Generate Storage Path & Open File
fileName := deriveBaseFileName(metadataInfo)
safePath := filepath.Join(api.cfg.DataPath, "documents", fileName)
// Open Destination File
destFile, err := os.Create(safePath)
if err != nil {
log.Error("Dest File Error: ", err)
appErrorPage(c, http.StatusInternalServerError, "Unable to save file.")
log.Errorf("unable to open destination file: %v", err)
appErrorPage(c, http.StatusInternalServerError, "Unable to open destination file")
return
}
defer destFile.Close()
// Copy File
if _, err = io.Copy(destFile, tempFile); err != nil {
log.Error("Copy Temp File Error: ", err)
appErrorPage(c, http.StatusInternalServerError, "Unable to save file.")
log.Errorf("unable to save file: %v", err)
appErrorPage(c, http.StatusInternalServerError, "Unable to save file")
return
}
// Upsert Document
if _, err = api.db.Queries.UpsertDocument(api.db.Ctx, database.UpsertDocumentParams{
ID: partialMD5,
ID: *metadataInfo.PartialMD5,
Title: metadataInfo.Title,
Author: metadataInfo.Author,
Description: metadataInfo.Description,
Words: &wordCount,
Md5: fileHash,
Md5: metadataInfo.MD5,
Words: metadataInfo.WordCount,
Filepath: &fileName,
// TODO (BasePath):
// - Should be current config directory
}); err != nil {
log.Error("UpsertDocument DB Error: ", err)
log.Errorf("UpsertDocument DB Error: %v", err)
appErrorPage(c, http.StatusInternalServerError, fmt.Sprintf("UpsertDocument DB Error: %v", err))
return
}
c.Redirect(http.StatusFound, fmt.Sprintf("./documents/%s", partialMD5))
c.Redirect(http.StatusFound, fmt.Sprintf("./documents/%s", *metadataInfo.PartialMD5))
}
func (api *API) appEditDocument(c *gin.Context) {
var rDocID requestDocumentID
if err := c.ShouldBindUri(&rDocID); err != nil {
log.Error("Invalid URI Bind")
appErrorPage(c, http.StatusNotFound, "Invalid document.")
appErrorPage(c, http.StatusNotFound, "Invalid document")
return
}
var rDocEdit requestDocumentEdit
if err := c.ShouldBind(&rDocEdit); err != nil {
log.Error("Invalid Form Bind")
appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values.")
appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values")
return
}
@ -586,7 +525,7 @@ func (api *API) appEditDocument(c *gin.Context) {
rDocEdit.CoverGBID == nil &&
rDocEdit.CoverFile == nil {
log.Error("Missing Form Values")
appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values.")
appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values")
return
}
@ -600,14 +539,14 @@ func (api *API) appEditDocument(c *gin.Context) {
uploadedFile, err := rDocEdit.CoverFile.Open()
if err != nil {
log.Error("File Error")
appErrorPage(c, http.StatusInternalServerError, "Unable to open file.")
appErrorPage(c, http.StatusInternalServerError, "Unable to open file")
return
}
fileMime, err := mimetype.DetectReader(uploadedFile)
if err != nil {
log.Error("MIME Error")
appErrorPage(c, http.StatusInternalServerError, "Unable to detect filetype.")
appErrorPage(c, http.StatusInternalServerError, "Unable to detect filetype")
return
}
fileExtension := fileMime.Extension()
@ -615,7 +554,7 @@ func (api *API) appEditDocument(c *gin.Context) {
// Validate Extension
if !slices.Contains([]string{".jpg", ".png"}, fileExtension) {
log.Error("Invalid FileType: ", fileExtension)
appErrorPage(c, http.StatusBadRequest, "Invalid filetype.")
appErrorPage(c, http.StatusBadRequest, "Invalid filetype")
return
}
@ -627,7 +566,7 @@ func (api *API) appEditDocument(c *gin.Context) {
err = c.SaveUploadedFile(rDocEdit.CoverFile, safePath)
if err != nil {
log.Error("File Error: ", err)
appErrorPage(c, http.StatusInternalServerError, "Unable to save file.")
appErrorPage(c, http.StatusInternalServerError, "Unable to save file")
return
}
@ -663,7 +602,7 @@ func (api *API) appDeleteDocument(c *gin.Context) {
var rDocID requestDocumentID
if err := c.ShouldBindUri(&rDocID); err != nil {
log.Error("Invalid URI Bind")
appErrorPage(c, http.StatusNotFound, "Invalid document.")
appErrorPage(c, http.StatusNotFound, "Invalid document")
return
}
changed, err := api.db.Queries.DeleteDocument(api.db.Ctx, rDocID.DocumentID)
@ -674,7 +613,7 @@ func (api *API) appDeleteDocument(c *gin.Context) {
}
if changed == 0 {
log.Error("DeleteDocument DB Error")
appErrorPage(c, http.StatusNotFound, "Invalid document.")
appErrorPage(c, http.StatusNotFound, "Invalid document")
return
}
@ -685,14 +624,14 @@ func (api *API) appIdentifyDocument(c *gin.Context) {
var rDocID requestDocumentID
if err := c.ShouldBindUri(&rDocID); err != nil {
log.Error("Invalid URI Bind")
appErrorPage(c, http.StatusNotFound, "Invalid document.")
appErrorPage(c, http.StatusNotFound, "Invalid document")
return
}
var rDocIdentify requestDocumentIdentify
if err := c.ShouldBind(&rDocIdentify); err != nil {
log.Error("Invalid Form Bind")
appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values.")
appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values")
return
}
@ -710,7 +649,7 @@ func (api *API) appIdentifyDocument(c *gin.Context) {
// Validate Values
if rDocIdentify.ISBN == nil && rDocIdentify.Title == nil && rDocIdentify.Author == nil {
log.Error("Invalid Form")
appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values.")
appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values")
return
}
@ -718,7 +657,7 @@ func (api *API) appIdentifyDocument(c *gin.Context) {
templateVars, auth := api.getBaseTemplateVars("document", c)
// Get Metadata
metadataResults, err := metadata.SearchMetadata(metadata.GBOOK, metadata.MetadataInfo{
metadataResults, err := metadata.SearchMetadata(metadata.SOURCE_GBOOK, metadata.MetadataInfo{
Title: rDocIdentify.Title,
Author: rDocIdentify.Author,
ISBN10: rDocIdentify.ISBN,
@ -767,7 +706,7 @@ func (api *API) appSaveNewDocument(c *gin.Context) {
var rDocAdd requestDocumentAdd
if err := c.ShouldBind(&rDocAdd); err != nil {
log.Error("Invalid Form Bind")
appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values.")
appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values")
return
}
@ -845,7 +784,7 @@ func (api *API) appSaveNewDocument(c *gin.Context) {
fileName = strings.ReplaceAll(fileName, "/", "")
// Derive & Sanitize File Name
fileName = "." + filepath.Clean(fmt.Sprintf("/%s [%s]%s", fileName, partialMD5, fileExtension))
fileName = "." + filepath.Clean(fmt.Sprintf("/%s [%s]%s", fileName, *partialMD5, fileExtension))
// Open Source File
sourceFile, err := os.Open(tempFilePath)
@ -901,12 +840,12 @@ func (api *API) appSaveNewDocument(c *gin.Context) {
// Upsert Document
if _, err = api.db.Queries.UpsertDocument(api.db.Ctx, database.UpsertDocumentParams{
ID: partialMD5,
ID: *partialMD5,
Title: rDocAdd.Title,
Author: rDocAdd.Author,
Md5: fileHash,
Filepath: &fileName,
Words: &wordCount,
Words: wordCount,
}); err != nil {
log.Error("UpsertDocument DB Error: ", err)
sendDownloadMessage("Unable to save to database", gin.H{"Error": true})
@ -917,7 +856,7 @@ func (api *API) appSaveNewDocument(c *gin.Context) {
sendDownloadMessage("Download Success", gin.H{
"Progress": 100,
"ButtonText": "Go to Book",
"ButtonHref": fmt.Sprintf("./documents/%s", partialMD5),
"ButtonHref": fmt.Sprintf("./documents/%s", *partialMD5),
})
}
@ -925,14 +864,14 @@ func (api *API) appEditSettings(c *gin.Context) {
var rUserSettings requestSettingsEdit
if err := c.ShouldBind(&rUserSettings); err != nil {
log.Error("Invalid Form Bind")
appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values.")
appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values")
return
}
// Validate Something Exists
if rUserSettings.Password == nil && rUserSettings.NewPassword == nil && rUserSettings.TimeOffset == nil {
log.Error("Missing Form Values")
appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values.")
appErrorPage(c, http.StatusBadRequest, "Invalid or missing form values")
return
}
@ -1023,7 +962,7 @@ func (api *API) getDocumentsWordCount(documents []database.GetDocumentsWithStats
} else {
if _, err := qtx.UpsertDocument(api.db.Ctx, database.UpsertDocumentParams{
ID: item.ID,
Words: &wordCount,
Words: wordCount,
}); err != nil {
log.Error("UpsertDocument DB Error: ", err)
return err

View File

@ -95,7 +95,7 @@ func (api *API) createGetCoverHandler(errorFunc func(*gin.Context, int, string))
var coverFile string = "UNKNOWN"
// Identify Documents & Save Covers
metadataResults, err := metadata.SearchMetadata(metadata.GBOOK, metadata.MetadataInfo{
metadataResults, err := metadata.SearchMetadata(metadata.SOURCE_GBOOK, metadata.MetadataInfo{
Title: document.Title,
Author: document.Author,
})

View File

@ -10,13 +10,10 @@ import (
"net/http"
"os"
"path/filepath"
"strings"
"time"
"github.com/gabriel-vasile/mimetype"
"github.com/gin-gonic/gin"
log "github.com/sirupsen/logrus"
"golang.org/x/exp/slices"
"reichard.io/antholume/database"
"reichard.io/antholume/metadata"
)
@ -456,21 +453,11 @@ func (api *API) koUploadExistingDocument(c *gin.Context) {
return
}
// Open Form File
fileData, err := c.FormFile("file")
if err != nil {
log.Error("File Error:", err)
apiErrorPage(c, http.StatusBadRequest, "File Error")
return
}
// Validate Type & Derive Extension on MIME
uploadedFile, err := fileData.Open()
fileMime, err := mimetype.DetectReader(uploadedFile)
fileExtension := fileMime.Extension()
if !slices.Contains([]string{".epub", ".html"}, fileExtension) {
log.Error("Invalid FileType:", fileExtension)
apiErrorPage(c, http.StatusBadRequest, "Invalid Filetype")
apiErrorPage(c, http.StatusBadRequest, "File error")
return
}
@ -482,25 +469,29 @@ func (api *API) koUploadExistingDocument(c *gin.Context) {
return
}
// Open File
uploadedFile, err := fileData.Open()
if err != nil {
log.Error("Unable to open file")
apiErrorPage(c, http.StatusBadRequest, "Unable to open file")
return
}
// Check Support
docType, err := metadata.GetDocumentTypeReader(uploadedFile)
if err != nil {
log.Error("Unsupported file")
apiErrorPage(c, http.StatusBadRequest, "Unsupported file")
return
}
// Derive Filename
var fileName string
if document.Author != nil {
fileName = fileName + *document.Author
} else {
fileName = fileName + "Unknown"
}
if document.Title != nil {
fileName = fileName + " - " + *document.Title
} else {
fileName = fileName + " - Unknown"
}
// Remove Slashes
fileName = strings.ReplaceAll(fileName, "/", "")
// Derive & Sanitize File Name
fileName = "." + filepath.Clean(fmt.Sprintf("/%s [%s]%s", fileName, document.ID, fileExtension))
fileName := deriveBaseFileName(&metadata.MetadataInfo{
Type: *docType,
PartialMD5: &document.ID,
Title: document.Title,
Author: document.Author,
})
// Generate Storage Path
safePath := filepath.Join(api.cfg.DataPath, "documents", fileName)
@ -516,28 +507,20 @@ func (api *API) koUploadExistingDocument(c *gin.Context) {
}
}
// Get MD5 Hash
fileHash, err := getFileMD5(safePath)
// Acquire Metadata
metadataInfo, err := metadata.GetMetadata(safePath)
if err != nil {
log.Error("Hash Failure:", err)
apiErrorPage(c, http.StatusBadRequest, "File Error")
return
}
// Get Word Count
wordCount, err := metadata.GetWordCount(safePath)
if err != nil {
log.Error("Word Count Failure:", err)
apiErrorPage(c, http.StatusBadRequest, "File Error")
log.Errorf("Unable to acquire metadata: %v", err)
apiErrorPage(c, http.StatusBadRequest, "Unable to acquire metadata")
return
}
// Upsert Document
if _, err = api.db.Queries.UpsertDocument(api.db.Ctx, database.UpsertDocumentParams{
ID: document.ID,
Md5: fileHash,
Md5: metadataInfo.MD5,
Words: metadataInfo.WordCount,
Filepath: &fileName,
Words: &wordCount,
}); err != nil {
log.Error("UpsertDocument DB Error:", err)
apiErrorPage(c, http.StatusBadRequest, "Document Error")

View File

@ -4,10 +4,13 @@ import (
"errors"
"fmt"
"math"
"path/filepath"
"reflect"
"strings"
"reichard.io/antholume/database"
"reichard.io/antholume/graph"
"reichard.io/antholume/metadata"
)
type UTCOffset struct {
@ -144,3 +147,22 @@ func fields(value interface{}) (map[string]interface{}, error) {
}
return m, nil
}
func deriveBaseFileName(metadataInfo *metadata.MetadataInfo) string {
// Derive New FileName
var newFileName string
if *metadataInfo.Author != "" {
newFileName = newFileName + *metadataInfo.Author
} else {
newFileName = newFileName + "Unknown"
}
if *metadataInfo.Title != "" {
newFileName = newFileName + " - " + *metadataInfo.Title
} else {
newFileName = newFileName + " - Unknown"
}
// Remove Slashes
fileName := strings.ReplaceAll(newFileName, "/", "")
return "." + filepath.Clean(fmt.Sprintf("/%s [%s]%s", fileName, *metadataInfo.PartialMD5, metadataInfo.Type))
}

View File

@ -1,12 +1,35 @@
package api
import "testing"
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestNiceSeconds(t *testing.T) {
want := "22d 7h 39m 31s"
nice := niceSeconds(1928371)
wantOne := "22d 7h 39m 31s"
wantNA := "N/A"
if nice != want {
t.Fatalf(`Expected: %v, Got: %v`, want, nice)
}
niceOne := niceSeconds(1928371)
niceNA := niceSeconds(0)
assert.Equal(t, wantOne, niceOne, "should be nice seconds")
assert.Equal(t, wantNA, niceNA, "should be nice NA")
}
func TestNiceNumbers(t *testing.T) {
wantMillions := "198M"
wantThousands := "19.8k"
wantThousandsTwo := "1.98k"
wantZero := "0"
niceMillions := niceNumbers(198236461)
niceThousands := niceNumbers(19823)
niceThousandsTwo := niceNumbers(1984)
niceZero := niceNumbers(0)
assert.Equal(t, wantMillions, niceMillions, "should be nice millions")
assert.Equal(t, wantThousands, niceThousands, "should be nice thousands")
assert.Equal(t, wantThousandsTwo, niceThousandsTwo, "should be nice thousands")
assert.Equal(t, wantZero, niceZero, "should be nice zero")
}

File diff suppressed because one or more lines are too long

View File

@ -1,4 +1,4 @@
<!DOCTYPE html>
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />

View File

@ -97,16 +97,18 @@ class EBookReader {
flow: "paginated",
width: "100%",
height: "100%",
allowScriptedContent: true,
});
// Setup Reader
this.book.ready.then(this.setupReader.bind(this));
// Initialize
this.initCSP();
this.initDevice();
this.initWakeLock();
this.initThemes();
this.initRenditionListeners();
this.initViewerListeners();
this.initDocumentListeners();
}
@ -279,6 +281,36 @@ class EBookReader {
);
}
/**
* EpubJS will set iframe sandbox when settings "allowScriptedContent: false".
* However, Safari completely blocks us from attaching listeners to the iframe
* document. So instead we just inject a restrictive CSP rule.
*
* This effectively blocks all script content within the iframe while still
* allowing us to attach listeners to the iframe document.
**/
initCSP() {
// Derive CSP Host
var protocol = document.location.protocol;
var host = document.location.host;
var cspURL = `${protocol}//${host}`;
// Add CSP Policy
this.book.spine.hooks.content.register((output, section) => {
let cspWrapper = document.createElement("div");
cspWrapper.innerHTML = `
<meta
http-equiv="Content-Security-Policy"
content="require-trusted-types-for 'script';
style-src 'self' blob: 'unsafe-inline' ${cspURL};
object-src 'none';
script-src 'none';"
>`;
let cspMeta = cspWrapper.children[0];
output.head.append(cspMeta);
});
}
/**
* Set theme & meta theme color
**/
@ -371,9 +403,9 @@ class EBookReader {
}
/**
* Rendition hooks
* Viewer Listeners
**/
initRenditionListeners() {
initViewerListeners() {
/**
* Initiate the debounce when the given function returns true.
* Don't run it again until the timeout lapses.
@ -401,56 +433,17 @@ class EBookReader {
let bottomBar = document.querySelector("#bottom-bar");
// Local Functions
let getCFIFromXPath = this.getCFIFromXPath.bind(this);
let setPosition = this.setPosition.bind(this);
let nextPage = this.nextPage.bind(this);
let prevPage = this.prevPage.bind(this);
let saveSettings = this.saveSettings.bind(this);
// Local Vars
let readerSettings = this.readerSettings;
let bookState = this.bookState;
this.rendition.hooks.render.register(function (doc, data) {
let renderDoc = doc.document;
// ------------------------------------------------ //
// ---------------- Wake Lock Hack ---------------- //
// ------------------------------------------------ //
let wakeLockListener = function () {
doc.window.parent.document.dispatchEvent(new CustomEvent("wakelock"));
};
renderDoc.addEventListener("click", wakeLockListener);
renderDoc.addEventListener("gesturechange", wakeLockListener);
renderDoc.addEventListener("touchstart", wakeLockListener);
// ------------------------------------------------ //
// --------------- Swipe Pagination --------------- //
// ----------------- Swipe Helpers ---------------- //
// ------------------------------------------------ //
let touchStartX,
touchStartY,
touchEndX,
touchEndY = undefined;
renderDoc.addEventListener(
"touchstart",
function (event) {
touchStartX = event.changedTouches[0].screenX;
touchStartY = event.changedTouches[0].screenY;
},
false,
);
renderDoc.addEventListener(
"touchend",
function (event) {
touchEndX = event.changedTouches[0].screenX;
touchEndY = event.changedTouches[0].screenY;
handleGesture(event);
},
false,
);
function handleGesture(event) {
let drasticity = 75;
@ -476,8 +469,32 @@ class EBookReader {
}
}
function handleSwipeDown() {
if (bottomBar.classList.contains("bottom-0"))
bottomBar.classList.remove("bottom-0");
else topBar.classList.add("top-0");
}
function handleSwipeUp() {
if (topBar.classList.contains("top-0")) topBar.classList.remove("top-0");
else bottomBar.classList.add("bottom-0");
}
this.rendition.hooks.render.register(function (doc, data) {
let renderDoc = doc.document;
// ------------------------------------------------ //
// --------------- Bottom & Top Bar --------------- //
// ---------------- Wake Lock Hack ---------------- //
// ------------------------------------------------ //
let wakeLockListener = function () {
renderDoc.dispatchEvent(new CustomEvent("wakelock"));
};
renderDoc.addEventListener("click", wakeLockListener);
renderDoc.addEventListener("gesturechange", wakeLockListener);
renderDoc.addEventListener("touchstart", wakeLockListener);
// ------------------------------------------------ //
// --------------- Bars & Page Turn --------------- //
// ------------------------------------------------ //
renderDoc.addEventListener(
"click",
@ -529,45 +546,25 @@ class EBookReader {
}, 400),
);
function handleSwipeDown() {
if (bottomBar.classList.contains("bottom-0"))
bottomBar.classList.remove("bottom-0");
else topBar.classList.add("top-0");
}
function handleSwipeUp() {
if (topBar.classList.contains("top-0"))
topBar.classList.remove("top-0");
else bottomBar.classList.add("bottom-0");
}
// ------------------------------------------------ //
// -------------- Keyboard Shortcuts -------------- //
// ------------------- Gestures ------------------- //
// ------------------------------------------------ //
renderDoc.addEventListener(
"keyup",
function (e) {
// Left Key (Previous Page)
if ((e.keyCode || e.which) == 37) {
prevPage();
}
// Right Key (Next Page)
if ((e.keyCode || e.which) == 39) {
nextPage();
}
// "t" Key (Theme Cycle)
if ((e.keyCode || e.which) == 84) {
let currentThemeIdx = THEMES.indexOf(
readerSettings.theme.colorScheme,
"touchstart",
function (event) {
touchStartX = event.changedTouches[0].screenX;
touchStartY = event.changedTouches[0].screenY;
},
false,
);
let colorScheme =
THEMES.length == currentThemeIdx + 1
? THEMES[0]
: THEMES[currentThemeIdx + 1];
setTheme({ colorScheme });
}
renderDoc.addEventListener(
"touchend",
function (event) {
touchEndX = event.changedTouches[0].screenX;
touchEndY = event.changedTouches[0].screenY;
handleGesture(event);
},
false,
);
@ -584,7 +581,9 @@ class EBookReader {
let nextPage = this.nextPage.bind(this);
let prevPage = this.prevPage.bind(this);
// Keyboard Shortcuts
// ------------------------------------------------ //
// -------------- Keyboard Shortcuts -------------- //
// ------------------------------------------------ //
document.addEventListener(
"keyup",
function (e) {

View File

@ -118,6 +118,7 @@ func (c *Config) EnsureDirectories() {
docDir := filepath.Join(c.DataPath, "documents")
coversDir := filepath.Join(c.DataPath, "covers")
backupDir := filepath.Join(c.DataPath, "backups")
os.Mkdir(docDir, 0755)
os.Mkdir(coversDir, 0755)
os.Mkdir(backupDir, 0755)

View File

@ -1,35 +1,37 @@
package config
import "testing"
import (
"runtime"
"testing"
"github.com/stretchr/testify/assert"
)
func TestLoadConfig(t *testing.T) {
conf := Load()
want := "sqlite"
if conf.DBType != want {
t.Fatalf(`Load().DBType = %q, want match for %#q, nil`, conf.DBType, want)
}
assert.Equal(t, "sqlite", conf.DBType)
}
func TestGetEnvDefault(t *testing.T) {
want := "def_val"
envDefault := getEnv("DEFAULT_TEST", want)
if envDefault != want {
t.Fatalf(`getEnv("DEFAULT_TEST", "def_val") = %q, want match for %#q, nil`, envDefault, want)
}
}
desiredValue := "def_val"
envDefault := getEnv("DEFAULT_TEST", desiredValue)
func TestGetEnvSet(t *testing.T) {
envDefault := getEnv("SET_TEST", "not_this")
want := "set_val"
if envDefault != want {
t.Fatalf(`getEnv("SET_TEST", "not_this") = %q, want match for %#q, nil`, envDefault, want)
}
assert.Equal(t, desiredValue, envDefault)
}
func TestTrimLowerString(t *testing.T) {
want := "trimtest"
output := trimLowerString(" trimTest ")
if output != want {
t.Fatalf(`trimLowerString(" trimTest ") = %q, want match for %#q, nil`, output, want)
}
desiredValue := "trimtest"
outputValue := trimLowerString(" trimTest ")
assert.Equal(t, desiredValue, outputValue)
}
func TestPrettyCaller(t *testing.T) {
p, _, _, _ := runtime.Caller(0)
result := runtime.CallersFrames([]uintptr{p})
f, _ := result.Next()
functionName, fileName := prettyCaller(&f)
assert.Equal(t, "TestPrettyCaller", functionName, "should have current function name")
assert.Equal(t, "config/config_test.go@30", fileName, "should have current file path and line number")
}

View File

@ -5,6 +5,7 @@ import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"reichard.io/antholume/config"
"reichard.io/antholume/utils"
)
@ -28,9 +29,7 @@ func TestNewMgr(t *testing.T) {
}
dbm := NewMgr(&cfg)
if dbm == nil {
t.Fatalf(`Expected: *DBManager, Got: nil`)
}
assert.NotNil(t, dbm, "should not be nil dbm")
t.Run("Database", func(t *testing.T) {
dt := databaseTest{t, dbm}
@ -46,9 +45,7 @@ func (dt *databaseTest) TestUser() {
dt.Run("User", func(t *testing.T) {
// Generate Auth Hash
rawAuthHash, err := utils.GenerateToken(64)
if err != nil {
t.Fatalf(`Expected: %v, Got: %v, Error: %v`, nil, err, err)
}
assert.Nil(t, err, "should be nil err")
authHash := fmt.Sprintf("%x", rawAuthHash)
changed, err := dt.dbm.Queries.CreateUser(dt.dbm.Ctx, CreateUserParams{
@ -57,14 +54,13 @@ func (dt *databaseTest) TestUser() {
AuthHash: &authHash,
})
if err != nil || changed != 1 {
t.Fatalf(`Expected: %v, Got: %v, Error: %v`, 1, changed, err)
}
assert.Nil(t, err, "should be nil err")
assert.Equal(t, int64(1), changed)
user, err := dt.dbm.Queries.GetUser(dt.dbm.Ctx, userID)
if err != nil || *user.Pass != userPass {
t.Fatalf(`Expected: %v, Got: %v, Error: %v`, userPass, *user.Pass, err)
}
assert.Nil(t, err, "should be nil err")
assert.Equal(t, userPass, *user.Pass)
})
}

4
go.mod
View File

@ -27,6 +27,7 @@ require (
github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect
github.com/chenzhuoyu/iasm v0.9.1 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/gin-contrib/sse v0.1.0 // indirect
github.com/go-playground/locales v0.14.1 // indirect
@ -39,6 +40,7 @@ require (
github.com/gorilla/securecookie v1.1.2 // indirect
github.com/gorilla/sessions v1.2.2 // indirect
github.com/itchyny/timefmt-go v0.1.5 // indirect
github.com/jarcoal/httpmock v1.3.1 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
github.com/klauspost/cpuid/v2 v2.2.6 // indirect
@ -47,9 +49,11 @@ require (
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/pelletier/go-toml/v2 v2.1.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/sethvargo/go-retry v0.2.4 // indirect
github.com/stretchr/testify v1.8.4 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/ugorji/go/codec v1.2.12 // indirect
github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e // indirect

2
go.sum
View File

@ -127,6 +127,8 @@ github.com/jackc/pgx/v5 v5.5.1 h1:5I9etrGkLrN+2XPCsi6XLlV5DITbSL/xBZdmAxFcXPI=
github.com/jackc/pgx/v5 v5.5.1/go.mod h1:Ig06C2Vu0t5qXC60W8sqIthScaEnFvojjj9dSljmHRA=
github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/jarcoal/httpmock v1.3.1 h1:iUx3whfZWVf3jT01hQTO/Eo5sAYtB2/rqaUuOtpInww=
github.com/jarcoal/httpmock v1.3.1/go.mod h1:3yb8rc4BI7TCBhFY8ng0gjuLKJNquuDNiPaZjnENuYg=
github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4=
github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak=
github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4=

View File

@ -0,0 +1,110 @@
{
"kind": "books#volume",
"id": "ZxwpakTv_MIC",
"etag": "mhqr3GsebaQ",
"selfLink": "https://www.googleapis.com/books/v1/volumes/ZxwpakTv_MIC",
"volumeInfo": {
"title": "Alice in Wonderland",
"authors": [
"Lewis Carroll"
],
"publisher": "The Floating Press",
"publishedDate": "2009-01-01",
"description": "Alice in Wonderland (also known as Alice's Adventures in Wonderland), from 1865, is the peculiar and imaginative tale of a girl who falls down a rabbit-hole into a bizarre world of eccentric and unusual creatures. Lewis Carroll's prominent example of the genre of \"literary nonsense\" has endured in popularity with its clever way of playing with logic and a narrative structure that has influence generations of fiction writing.",
"industryIdentifiers": [
{
"type": "ISBN_10",
"identifier": "1877527815"
},
{
"type": "ISBN_13",
"identifier": "9781877527814"
}
],
"readingModes": {
"text": true,
"image": false
},
"pageCount": 104,
"printedPageCount": 112,
"printType": "BOOK",
"categories": [
"Fiction / Classics",
"Juvenile Fiction / General"
],
"averageRating": 5,
"ratingsCount": 1,
"maturityRating": "NOT_MATURE",
"allowAnonLogging": true,
"contentVersion": "0.2.3.0.preview.2",
"panelizationSummary": {
"containsEpubBubbles": false,
"containsImageBubbles": false
},
"imageLinks": {
"smallThumbnail": "http://books.google.com/books/content?id=ZxwpakTv_MIC&printsec=frontcover&img=1&zoom=5&edge=curl&imgtk=AFLRE71e5b-TeAKTiPSvXNUPeUi8rItzur2xSzwH8QU3qjKH0A2opmoq1o5I9RqJFt1BtcCCqILhnYRcB2aFLJmEvom11gx3Qn3PNN1iBLj2H5y2JHjM8wIwGT7iWFQmEn0Od7s6sOdk&source=gbs_api",
"thumbnail": "http://books.google.com/books/content?id=ZxwpakTv_MIC&printsec=frontcover&img=1&zoom=1&edge=curl&imgtk=AFLRE70QORt9J_DmKJgfyf9UEjQkdDMZ0qAu0GP315a1Q4CRS3snEjKnJJO2fYFdxjMwsSpmHoXDFPZbsy4gw-kMvF7lL8LtwxGbJGlfETHw_jbQBKBlKTrneK4XFvvV-EXNrZRgylxj&source=gbs_api",
"small": "http://books.google.com/books/content?id=ZxwpakTv_MIC&printsec=frontcover&img=1&zoom=2&edge=curl&imgtk=AFLRE70r1pAUt6VhuEEW8vXFhu8LvKln3yj0mdlaWPO4ZQuODLFQnH0fTebKMMX4ANR5i4PtC0oaI48XkwF-EdzlEM1WmUcR5383N4kRMXcta_i9nmb2y38dnh3hObwQW5VoAxbc9psn&source=gbs_api",
"medium": "http://books.google.com/books/content?id=ZxwpakTv_MIC&printsec=frontcover&img=1&zoom=3&edge=curl&imgtk=AFLRE7019EVuXvhzbhmtbz1QFh-ajB6kTKRHGhqijFf8big_GPRMMdpCdKlklFbkCfXvy8F64t5NKlThUHb3tFP-51bbDXkrVErFbCqKGzGnDSSm8cewqT8HiYDNHqn0hXYnuYvN4vYf&source=gbs_api",
"large": "http://books.google.com/books/content?id=ZxwpakTv_MIC&printsec=frontcover&img=1&zoom=4&edge=curl&imgtk=AFLRE72I15XZqp_8c8BAj4EskxkdC6nQz8F0Fs6VJhkykwIqfjzwuM34tUSQa3UnMGbx-UYjZjSLmCNFlePS8aR7yy-0UP9BRnYD-h5Qbesnnt_xdOb3u7Wdiobi6VbciNCBwUwbCyeH&source=gbs_api",
"extraLarge": "http://books.google.com/books/content?id=ZxwpakTv_MIC&printsec=frontcover&img=1&zoom=6&edge=curl&imgtk=AFLRE70rC6ktY6U0K_hqG1HxPl_9hMjpKb10p9DryVIwQgUjoJfWQOjpNA3EQ-5yk167yYDlO27gylqNAdJBYWu7ZHr3GuqkjTDpXjDvzBBppVyWaVNxKwhOz3gfJ-gzM6cC4kLHP26R&source=gbs_api"
},
"language": "en",
"previewLink": "http://books.google.com/books?id=ZxwpakTv_MIC&hl=&source=gbs_api",
"infoLink": "https://play.google.com/store/books/details?id=ZxwpakTv_MIC&source=gbs_api",
"canonicalVolumeLink": "https://play.google.com/store/books/details?id=ZxwpakTv_MIC"
},
"layerInfo": {
"layers": [
{
"layerId": "geo",
"volumeAnnotationsVersion": "2"
}
]
},
"saleInfo": {
"country": "US",
"saleability": "FOR_SALE",
"isEbook": true,
"listPrice": {
"amount": 3.99,
"currencyCode": "USD"
},
"retailPrice": {
"amount": 3.99,
"currencyCode": "USD"
},
"buyLink": "https://play.google.com/store/books/details?id=ZxwpakTv_MIC&rdid=book-ZxwpakTv_MIC&rdot=1&source=gbs_api",
"offers": [
{
"finskyOfferType": 1,
"listPrice": {
"amountInMicros": 3990000,
"currencyCode": "USD"
},
"retailPrice": {
"amountInMicros": 3990000,
"currencyCode": "USD"
},
"giftable": true
}
]
},
"accessInfo": {
"country": "US",
"viewability": "PARTIAL",
"embeddable": true,
"publicDomain": false,
"textToSpeechPermission": "ALLOWED",
"epub": {
"isAvailable": true,
"acsTokenLink": "http://books.google.com/books/download/Alice_in_Wonderland-sample-epub.acsm?id=ZxwpakTv_MIC&format=epub&output=acs4_fulfillment_token&dl_type=sample&source=gbs_api"
},
"pdf": {
"isAvailable": false
},
"webReaderLink": "http://play.google.com/books/reader?id=ZxwpakTv_MIC&hl=&source=gbs_api",
"accessViewStatus": "SAMPLE",
"quoteSharingAllowed": false
}
}

View File

@ -0,0 +1,105 @@
{
"kind": "books#volumes",
"totalItems": 1,
"items": [
{
"kind": "books#volume",
"id": "ZxwpakTv_MIC",
"etag": "F2eR9VV6VwQ",
"selfLink": "https://www.googleapis.com/books/v1/volumes/ZxwpakTv_MIC",
"volumeInfo": {
"title": "Alice in Wonderland",
"authors": [
"Lewis Carroll"
],
"publisher": "The Floating Press",
"publishedDate": "2009-01-01",
"description": "Alice in Wonderland (also known as Alice's Adventures in Wonderland), from 1865, is the peculiar and imaginative tale of a girl who falls down a rabbit-hole into a bizarre world of eccentric and unusual creatures. Lewis Carroll's prominent example of the genre of \"literary nonsense\" has endured in popularity with its clever way of playing with logic and a narrative structure that has influence generations of fiction writing.",
"industryIdentifiers": [
{
"type": "ISBN_13",
"identifier": "9781877527814"
},
{
"type": "ISBN_10",
"identifier": "1877527815"
}
],
"readingModes": {
"text": true,
"image": false
},
"pageCount": 104,
"printType": "BOOK",
"categories": [
"Fiction"
],
"averageRating": 5,
"ratingsCount": 1,
"maturityRating": "NOT_MATURE",
"allowAnonLogging": true,
"contentVersion": "0.2.3.0.preview.2",
"panelizationSummary": {
"containsEpubBubbles": false,
"containsImageBubbles": false
},
"imageLinks": {
"smallThumbnail": "http://books.google.com/books/content?id=ZxwpakTv_MIC&printsec=frontcover&img=1&zoom=5&edge=curl&source=gbs_api",
"thumbnail": "http://books.google.com/books/content?id=ZxwpakTv_MIC&printsec=frontcover&img=1&zoom=1&edge=curl&source=gbs_api"
},
"language": "en",
"previewLink": "http://books.google.com/books?id=ZxwpakTv_MIC&printsec=frontcover&dq=isbn:1877527815&hl=&cd=1&source=gbs_api",
"infoLink": "https://play.google.com/store/books/details?id=ZxwpakTv_MIC&source=gbs_api",
"canonicalVolumeLink": "https://play.google.com/store/books/details?id=ZxwpakTv_MIC"
},
"saleInfo": {
"country": "US",
"saleability": "FOR_SALE",
"isEbook": true,
"listPrice": {
"amount": 3.99,
"currencyCode": "USD"
},
"retailPrice": {
"amount": 3.99,
"currencyCode": "USD"
},
"buyLink": "https://play.google.com/store/books/details?id=ZxwpakTv_MIC&rdid=book-ZxwpakTv_MIC&rdot=1&source=gbs_api",
"offers": [
{
"finskyOfferType": 1,
"listPrice": {
"amountInMicros": 3990000,
"currencyCode": "USD"
},
"retailPrice": {
"amountInMicros": 3990000,
"currencyCode": "USD"
},
"giftable": true
}
]
},
"accessInfo": {
"country": "US",
"viewability": "PARTIAL",
"embeddable": true,
"publicDomain": false,
"textToSpeechPermission": "ALLOWED",
"epub": {
"isAvailable": true,
"acsTokenLink": "http://books.google.com/books/download/Alice_in_Wonderland-sample-epub.acsm?id=ZxwpakTv_MIC&format=epub&output=acs4_fulfillment_token&dl_type=sample&source=gbs_api"
},
"pdf": {
"isAvailable": false
},
"webReaderLink": "http://play.google.com/books/reader?id=ZxwpakTv_MIC&hl=&source=gbs_api",
"accessViewStatus": "SAMPLE",
"quoteSharingAllowed": false
},
"searchInfo": {
"textSnippet": "Alice in Wonderland (also known as Alice&#39;s Adventures in Wonderland), from 1865, is the peculiar and imaginative tale of a girl who falls down a rabbit-hole into a bizarre world of eccentric and unusual creatures."
}
}
]
}

View File

@ -16,6 +16,7 @@ func getEPUBMetadata(filepath string) (*MetadataInfo, error) {
rf := rc.Rootfiles[0]
parsedMetadata := &MetadataInfo{
Type: TYPE_EPUB,
Title: &rf.Title,
Author: &rf.Creator,
Description: &rf.Description,

130
metadata/gbooks_test.go Normal file
View File

@ -0,0 +1,130 @@
package metadata
import (
_ "embed"
"encoding/json"
"fmt"
"net/http"
"net/url"
"regexp"
"strings"
"testing"
"github.com/jarcoal/httpmock"
"github.com/stretchr/testify/assert"
)
// const GBOOKS_QUERY_URL string = "https://www.googleapis.com/books/v1/volumes?q=%s"
// const GBOOKS_GBID_INFO_URL string = "https://www.googleapis.com/books/v1/volumes/%s"
// const GBOOKS_GBID_COVER_URL string = "https://books.google.com/books/content/images/frontcover/%s?fife=w480-h690"
//go:embed _test_files/gbooks_id_response.json
var idResp string
//go:embed _test_files/gbooks_query_response.json
var queryResp string
type details struct {
URLs []string
}
// Hook API Helper
func hookAPI() *details {
// Start HTTPMock
httpmock.Activate()
// Create details struct
d := &details{
URLs: []string{},
}
// Create Hook
matchRE := regexp.MustCompile(`^https://www\.googleapis\.com/books/v1/volumes.*`)
httpmock.RegisterRegexpResponder("GET", matchRE, func(req *http.Request) (*http.Response, error) {
// Append URL
d.URLs = append(d.URLs, req.URL.String())
// Get Raw Response
var rawResp string
if req.URL.Query().Get("q") != "" {
rawResp = queryResp
} else {
rawResp = idResp
}
// Convert to JSON Response
var responseData map[string]interface{}
json.Unmarshal([]byte(rawResp), &responseData)
// Return Response
return httpmock.NewJsonResponse(200, responseData)
})
return d
}
func TestGBooksGBIDMetadata(t *testing.T) {
hookDetails := hookAPI()
defer httpmock.DeactivateAndReset()
GBID := "ZxwpakTv_MIC"
expectedURL := fmt.Sprintf(GBOOKS_GBID_INFO_URL, GBID)
metadataResp, err := getGBooksMetadata(MetadataInfo{ID: &GBID})
assert.Nil(t, err, "should not have error")
assert.Contains(t, hookDetails.URLs, expectedURL, "should have intercepted URL")
assert.Equal(t, 1, len(metadataResp), "should have single result")
mResult := metadataResp[0]
validateResult(t, &mResult)
}
func TestGBooksISBNQuery(t *testing.T) {
hookDetails := hookAPI()
defer httpmock.DeactivateAndReset()
ISBN10 := "1877527815"
expectedURL := fmt.Sprintf(GBOOKS_QUERY_URL, "isbn:"+ISBN10)
metadataResp, err := getGBooksMetadata(MetadataInfo{
ISBN10: &ISBN10,
})
assert.Nil(t, err, "should not have error")
assert.Contains(t, hookDetails.URLs, expectedURL, "should have intercepted URL")
assert.Equal(t, 1, len(metadataResp), "should have single result")
mResult := metadataResp[0]
validateResult(t, &mResult)
}
func TestGBooksTitleQuery(t *testing.T) {
hookDetails := hookAPI()
defer httpmock.DeactivateAndReset()
title := "Alice in Wonderland 1877527815"
expectedURL := fmt.Sprintf(GBOOKS_QUERY_URL, url.QueryEscape(strings.TrimSpace(title)))
metadataResp, err := getGBooksMetadata(MetadataInfo{
Title: &title,
})
assert.Nil(t, err, "should not have error")
assert.Contains(t, hookDetails.URLs, expectedURL, "should have intercepted URL")
assert.NotEqual(t, 0, len(metadataResp), "should not have no results")
mResult := metadataResp[0]
validateResult(t, &mResult)
}
func validateResult(t *testing.T, m *MetadataInfo) {
expectedTitle := "Alice in Wonderland"
expectedAuthor := "Lewis Carroll"
expectedDesc := "Alice in Wonderland (also known as Alice's Adventures in Wonderland), from 1865, is the peculiar and imaginative tale of a girl who falls down a rabbit-hole into a bizarre world of eccentric and unusual creatures. Lewis Carroll's prominent example of the genre of \"literary nonsense\" has endured in popularity with its clever way of playing with logic and a narrative structure that has influence generations of fiction writing."
expectedISBN10 := "1877527815"
expectedISBN13 := "9781877527814"
assert.Equal(t, expectedTitle, *m.Title, "should have title")
assert.Equal(t, expectedAuthor, *m.Author, "should have author")
assert.Equal(t, expectedDesc, *m.Description, "should have description")
assert.Equal(t, expectedISBN10, *m.ISBN10, "should have ISBN10")
assert.Equal(t, expectedISBN13, *m.ISBN13, "should have ISBN10")
}

View File

@ -1,76 +0,0 @@
//go:build integration
package metadata
import (
"testing"
)
func TestGBooksGBIDMetadata(t *testing.T) {
GBID := "ZxwpakTv_MIC"
metadataResp, err := getGBooksMetadata(MetadataInfo{
ID: &GBID,
})
if len(metadataResp) != 1 {
t.Fatalf(`Expected: %v, Got: %v, Error: %v`, 1, len(metadataResp), err)
}
mResult := metadataResp[0]
validateResult(&mResult, t)
}
func TestGBooksISBNQuery(t *testing.T) {
ISBN10 := "1877527815"
metadataResp, err := getGBooksMetadata(MetadataInfo{
ISBN10: &ISBN10,
})
if len(metadataResp) != 1 {
t.Fatalf(`Expected: %v, Got: %v, Error: %v`, 1, len(metadataResp), err)
}
mResult := metadataResp[0]
validateResult(&mResult, t)
}
func TestGBooksTitleQuery(t *testing.T) {
title := "Alice in Wonderland 1877527815"
metadataResp, err := getGBooksMetadata(MetadataInfo{
Title: &title,
})
if len(metadataResp) == 0 {
t.Fatalf(`Expected: %v, Got: %v, Error: %v`, "> 0", len(metadataResp), err)
}
mResult := metadataResp[0]
validateResult(&mResult, t)
}
func validateResult(m *MetadataInfo, t *testing.T) {
expect := "Lewis Carroll"
if *m.Author != expect {
t.Fatalf(`Expected: %v, Got: %v`, expect, *m.Author)
}
expect = "Alice in Wonderland"
if *m.Title != expect {
t.Fatalf(`Expected: %v, Got: %v`, expect, *m.Title)
}
expect = "Alice in Wonderland (also known as Alice's Adventures in Wonderland), from 1865, is the peculiar and imaginative tale of a girl who falls down a rabbit-hole into a bizarre world of eccentric and unusual creatures. Lewis Carroll's prominent example of the genre of \"literary nonsense\" has endured in popularity with its clever way of playing with logic and a narrative structure that has influence generations of fiction writing."
if *m.Description != expect {
t.Fatalf(`Expected: %v, Got: %v`, expect, *m.Description)
}
expect = "1877527815"
if *m.ISBN10 != expect {
t.Fatalf(`Expected: %v, Got: %v`, expect, *m.ISBN10)
}
expect = "9781877527814"
if *m.ISBN13 != expect {
t.Fatalf(`Expected: %v, Got: %v`, expect, *m.ISBN13)
}
}

View File

@ -3,27 +3,47 @@ package metadata
import (
"errors"
"fmt"
"io"
"path/filepath"
"github.com/gabriel-vasile/mimetype"
"reichard.io/antholume/utils"
)
type MetadataHandler func(string) (*MetadataInfo, error)
type DocumentType string
const (
TYPE_EPUB DocumentType = ".epub"
)
var extensionHandlerMap = map[DocumentType]MetadataHandler{
TYPE_EPUB: getEPUBMetadata,
}
type Source int
const (
GBOOK Source = iota
OLIB
SOURCE_GBOOK Source = iota
SOURCE_OLIB
)
type MetadataInfo struct {
ID *string
MD5 *string
PartialMD5 *string
WordCount *int64
Title *string
Author *string
Description *string
ISBN10 *string
ISBN13 *string
Type DocumentType
}
// Downloads the Google Books cover file and saves it to the provided directory.
func CacheCover(gbid string, coverDir string, documentID string, overwrite bool) (*string, error) {
// Get Filepath
coverFile := "." + filepath.Clean(fmt.Sprintf("/%s.jpg", documentID))
@ -39,11 +59,12 @@ func CacheCover(gbid string, coverDir string, documentID string, overwrite bool)
return &coverFile, nil
}
// Searches source for metadata based on the provided information.
func SearchMetadata(s Source, metadataSearch MetadataInfo) ([]MetadataInfo, error) {
switch s {
case GBOOK:
case SOURCE_GBOOK:
return getGBooksMetadata(metadataSearch)
case OLIB:
case SOURCE_OLIB:
return nil, errors.New("Not implemented")
default:
return nil, errors.New("Not implemented")
@ -51,32 +72,112 @@ func SearchMetadata(s Source, metadataSearch MetadataInfo) ([]MetadataInfo, erro
}
}
func GetWordCount(filepath string) (int64, error) {
fileMime, err := mimetype.DetectFile(filepath)
if err != nil {
return 0, err
}
if fileExtension := fileMime.Extension(); fileExtension == ".epub" {
totalWords, err := countEPUBWords(filepath)
if err != nil {
return 0, err
}
return totalWords, nil
} else {
return 0, errors.New("Invalid Extension")
}
}
func GetMetadata(filepath string) (*MetadataInfo, error) {
// Returns the word count of the provided filepath. An error will be returned
// if the file is not supported.
func GetWordCount(filepath string) (*int64, error) {
fileMime, err := mimetype.DetectFile(filepath)
if err != nil {
return nil, err
}
if fileExtension := fileMime.Extension(); fileExtension == ".epub" {
return getEPUBMetadata(filepath)
totalWords, err := countEPUBWords(filepath)
if err != nil {
return nil, err
}
return &totalWords, nil
} else {
return nil, errors.New("Invalid Extension")
return nil, fmt.Errorf("Invalid extension")
}
}
// Returns embedded metadata of the provided file. An error will be returned if
// the file is not supported.
func GetMetadata(filepath string) (*MetadataInfo, error) {
// Detect Extension Type
fileMime, err := mimetype.DetectFile(filepath)
if err != nil {
return nil, err
}
// Get Extension Type Metadata Handler
fileExtension := fileMime.Extension()
handler, ok := extensionHandlerMap[DocumentType(fileExtension)]
if !ok {
return nil, fmt.Errorf("invalid extension %s", fileExtension)
}
// Acquire Metadata
metadataInfo, err := handler(filepath)
// Calculate MD5 & Partial MD5
partialMD5, err := utils.CalculatePartialMD5(filepath)
if err != nil {
return nil, fmt.Errorf("unable to calculate partial MD5")
}
// Calculate Actual MD5
MD5, err := utils.CalculateMD5(filepath)
if err != nil {
return nil, fmt.Errorf("unable to calculate MD5")
}
// Calculate Word Count
wordCount, err := GetWordCount(filepath)
if err != nil {
return nil, fmt.Errorf("unable to calculate word count")
}
metadataInfo.WordCount = wordCount
metadataInfo.PartialMD5 = partialMD5
metadataInfo.MD5 = MD5
return metadataInfo, nil
}
// Returns the extension of the provided filepath (e.g. ".epub"). An error
// will be returned if the file is not supported.
func GetDocumentType(filepath string) (*DocumentType, error) {
// Detect Extension Type
fileMime, err := mimetype.DetectFile(filepath)
if err != nil {
return nil, err
}
// Detect
fileExtension := fileMime.Extension()
docType, ok := ParseDocumentType(fileExtension)
if !ok {
return nil, fmt.Errorf("filetype not supported")
}
return &docType, nil
}
// Returns the extension of the provided file reader (e.g. ".epub"). An error
// will be returned if the file is not supported.
func GetDocumentTypeReader(r io.Reader) (*DocumentType, error) {
// Detect Extension Type
fileMime, err := mimetype.DetectReader(r)
if err != nil {
return nil, err
}
// Detect
fileExtension := fileMime.Extension()
docType, ok := ParseDocumentType(fileExtension)
if !ok {
return nil, fmt.Errorf("filetype not supported")
}
return &docType, nil
}
// Given a filetype string, attempt to resolve a DocumentType
func ParseDocumentType(input string) (DocumentType, bool) {
validTypes := map[string]DocumentType{
string(TYPE_EPUB): TYPE_EPUB,
}
found, ok := validTypes[input]
return found, ok
}

View File

@ -1,36 +1,46 @@
package metadata
import (
"os"
"testing"
"github.com/stretchr/testify/assert"
)
func TestGetWordCount(t *testing.T) {
var want int64 = 30080
wordCount, err := countEPUBWords("../_test_files/alice.epub")
var desiredCount int64 = 30080
actualCount, err := countEPUBWords("../_test_files/alice.epub")
assert.Nil(t, err, "should have no error")
assert.Equal(t, desiredCount, actualCount, "should be correct word count")
if wordCount != want {
t.Fatalf(`Expected: %v, Got: %v, Error: %v`, want, wordCount, err)
}
}
func TestGetMetadata(t *testing.T) {
metadataInfo, err := getEPUBMetadata("../_test_files/alice.epub")
if err != nil {
t.Fatalf(`Expected: *MetadataInfo, Got: nil, Error: %v`, err)
}
desiredTitle := "Alice's Adventures in Wonderland / Illustrated by Arthur Rackham. With a Proem by Austin Dobson"
desiredAuthor := "Lewis Carroll"
desiredDescription := ""
want := "Alice's Adventures in Wonderland / Illustrated by Arthur Rackham. With a Proem by Austin Dobson"
if *metadataInfo.Title != want {
t.Fatalf(`Expected: %v, Got: %v, Error: %v`, want, *metadataInfo.Title, err)
}
metadataInfo, err := GetMetadata("../_test_files/alice.epub")
want = "Lewis Carroll"
if *metadataInfo.Author != want {
t.Fatalf(`Expected: %v, Got: %v, Error: %v`, want, *metadataInfo.Author, err)
}
want = ""
if *metadataInfo.Description != want {
t.Fatalf(`Expected: %v, Got: %v, Error: %v`, want, *metadataInfo.Description, err)
}
assert.Nil(t, err, "should have no error")
assert.Equal(t, desiredTitle, *metadataInfo.Title, "should be correct title")
assert.Equal(t, desiredAuthor, *metadataInfo.Author, "should be correct author")
assert.Equal(t, desiredDescription, *metadataInfo.Description, "should be correct author")
assert.Equal(t, TYPE_EPUB, metadataInfo.Type, "should be correct type")
}
func TestGetExtension(t *testing.T) {
docType, err := GetDocumentType("../_test_files/alice.epub")
assert.Nil(t, err, "should have no error")
assert.Equal(t, TYPE_EPUB, *docType)
}
func TestGetExtensionReader(t *testing.T) {
file, _ := os.Open("../_test_files/alice.epub")
docType, err := GetDocumentTypeReader(file)
assert.Nil(t, err, "should have no error")
assert.Equal(t, TYPE_EPUB, *docType)
}

View File

@ -8,8 +8,8 @@ import (
// Feed root element for acquisition or navigation feed
type Feed struct {
ID string `xml:"id,omitempty"`
XMLName xml.Name `xml:"feed"`
ID string `xml:"id,omitempty",`
Title string `xml:"title,omitempty"`
Updated time.Time `xml:"updated,omitempty"`
Entries []Entry `xml:"entry,omitempty"`

75
search/anna.go Normal file
View File

@ -0,0 +1,75 @@
package search
import (
"fmt"
"io"
"strings"
"github.com/PuerkitoBio/goquery"
)
func parseAnnasArchiveDownloadURL(body io.ReadCloser) (string, error) {
// Parse
defer body.Close()
doc, _ := goquery.NewDocumentFromReader(body)
// Return Download URL
downloadURL, exists := doc.Find("body > table > tbody > tr > td > a").Attr("href")
if exists == false {
return "", fmt.Errorf("Download URL not found")
}
// Possible Funky URL
downloadURL = strings.ReplaceAll(downloadURL, "\\", "/")
return downloadURL, nil
}
func parseAnnasArchive(body io.ReadCloser) ([]SearchItem, error) {
// Parse
defer body.Close()
doc, err := goquery.NewDocumentFromReader(body)
if err != nil {
return nil, err
}
// Normalize Results
var allEntries []SearchItem
doc.Find("form > div.w-full > div.w-full > div > div.justify-center").Each(func(ix int, rawBook *goquery.Selection) {
// Parse Details
details := rawBook.Find("div:nth-child(2) > div:nth-child(1)").Text()
detailsSplit := strings.Split(details, ", ")
// Invalid Details
if len(detailsSplit) < 3 {
return
}
language := detailsSplit[0]
fileType := detailsSplit[1]
fileSize := detailsSplit[2]
// Get Title & Author
title := rawBook.Find("h3").Text()
author := rawBook.Find("div:nth-child(2) > div:nth-child(4)").Text()
// Parse MD5
itemHref, _ := rawBook.Find("a").Attr("href")
hrefArray := strings.Split(itemHref, "/")
id := hrefArray[len(hrefArray)-1]
item := SearchItem{
ID: id,
Title: title,
Author: author,
Language: language,
FileType: fileType,
FileSize: fileSize,
}
allEntries = append(allEntries, item)
})
// Return Results
return allEntries, nil
}

42
search/goodreads.go Normal file
View File

@ -0,0 +1,42 @@
package search
import (
"io"
"github.com/PuerkitoBio/goquery"
)
func GoodReadsMostRead(c Cadence) ([]SearchItem, error) {
body, err := getPage("https://www.goodreads.com/book/most_read?category=all&country=US&duration=" + string(c))
if err != nil {
return nil, err
}
return parseGoodReads(body)
}
func parseGoodReads(body io.ReadCloser) ([]SearchItem, error) {
// Parse
defer body.Close()
doc, err := goquery.NewDocumentFromReader(body)
if err != nil {
return nil, err
}
// Normalize Results
var allEntries []SearchItem
doc.Find("[itemtype=\"http://schema.org/Book\"]").Each(func(ix int, rawBook *goquery.Selection) {
title := rawBook.Find(".bookTitle span").Text()
author := rawBook.Find(".authorName span").Text()
item := SearchItem{
Title: title,
Author: author,
}
allEntries = append(allEntries, item)
})
// Return Results
return allEntries, nil
}

123
search/libgen.go Normal file
View File

@ -0,0 +1,123 @@
package search
import (
"fmt"
"io"
"strings"
"time"
"github.com/PuerkitoBio/goquery"
)
func parseLibGenFiction(body io.ReadCloser) ([]SearchItem, error) {
// Parse
defer body.Close()
doc, err := goquery.NewDocumentFromReader(body)
if err != nil {
return nil, err
}
// Normalize Results
var allEntries []SearchItem
doc.Find("table.catalog tbody > tr").Each(func(ix int, rawBook *goquery.Selection) {
// Parse File Details
fileItem := rawBook.Find("td:nth-child(5)")
fileDesc := fileItem.Text()
fileDescSplit := strings.Split(fileDesc, "/")
fileType := strings.ToLower(strings.TrimSpace(fileDescSplit[0]))
fileSize := strings.TrimSpace(fileDescSplit[1])
// Parse Upload Date
uploadedRaw, _ := fileItem.Attr("title")
uploadedDateRaw := strings.Split(uploadedRaw, "Uploaded at ")[1]
uploadDate, _ := time.Parse("2006-01-02 15:04:05", uploadedDateRaw)
// Parse MD5
editHref, _ := rawBook.Find("td:nth-child(7) a").Attr("href")
hrefArray := strings.Split(editHref, "/")
id := hrefArray[len(hrefArray)-1]
// Parse Other Details
title := rawBook.Find("td:nth-child(3) p a").Text()
author := rawBook.Find(".catalog_authors li a").Text()
language := rawBook.Find("td:nth-child(4)").Text()
series := rawBook.Find("td:nth-child(2)").Text()
item := SearchItem{
ID: id,
Title: title,
Author: author,
Series: series,
Language: language,
FileType: fileType,
FileSize: fileSize,
UploadDate: uploadDate.Format(time.RFC3339),
}
allEntries = append(allEntries, item)
})
// Return Results
return allEntries, nil
}
func parseLibGenNonFiction(body io.ReadCloser) ([]SearchItem, error) {
// Parse
defer body.Close()
doc, err := goquery.NewDocumentFromReader(body)
if err != nil {
return nil, err
}
// Normalize Results
var allEntries []SearchItem
doc.Find("table.c tbody > tr:nth-child(n + 2)").Each(func(ix int, rawBook *goquery.Selection) {
// Parse Type & Size
fileSize := strings.ToLower(strings.TrimSpace(rawBook.Find("td:nth-child(8)").Text()))
fileType := strings.ToLower(strings.TrimSpace(rawBook.Find("td:nth-child(9)").Text()))
// Parse MD5
titleRaw := rawBook.Find("td:nth-child(3) [id]")
editHref, _ := titleRaw.Attr("href")
hrefArray := strings.Split(editHref, "?md5=")
id := hrefArray[1]
// Parse Other Details
title := titleRaw.Text()
author := rawBook.Find("td:nth-child(2)").Text()
language := rawBook.Find("td:nth-child(7)").Text()
series := rawBook.Find("td:nth-child(3) [href*='column=series']").Text()
item := SearchItem{
ID: id,
Title: title,
Author: author,
Series: series,
Language: language,
FileType: fileType,
FileSize: fileSize,
}
allEntries = append(allEntries, item)
})
// Return Results
return allEntries, nil
}
func parseLibGenDownloadURL(body io.ReadCloser) (string, error) {
// Parse
defer body.Close()
doc, _ := goquery.NewDocumentFromReader(body)
// Return Download URL
// downloadURL, _ := doc.Find("#download [href*=cloudflare]").Attr("href")
downloadURL, exists := doc.Find("#download h2 a").Attr("href")
if exists == false {
return "", fmt.Errorf("Download URL not found")
}
return downloadURL, nil
}

View File

@ -2,16 +2,13 @@ package search
import (
"crypto/tls"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"os"
"strings"
"time"
"github.com/PuerkitoBio/goquery"
log "github.com/sirupsen/logrus"
)
@ -102,14 +99,14 @@ func SaveBook(id string, source Source) (string, error) {
bookURL, err := def.parseDownloadFunc(body)
if err != nil {
log.Error("Parse Download URL Error: ", err)
return "", errors.New("Download Failure")
return "", fmt.Errorf("Download Failure")
}
// Create File
tempFile, err := os.CreateTemp("", "book")
if err != nil {
log.Error("File Create Error: ", err)
return "", errors.New("File Failure")
return "", fmt.Errorf("File Failure")
}
defer tempFile.Close()
@ -119,7 +116,7 @@ func SaveBook(id string, source Source) (string, error) {
if err != nil {
os.Remove(tempFile.Name())
log.Error("Book URL API Failure: ", err)
return "", errors.New("API Failure")
return "", fmt.Errorf("API Failure")
}
defer resp.Body.Close()
@ -129,20 +126,12 @@ func SaveBook(id string, source Source) (string, error) {
if err != nil {
os.Remove(tempFile.Name())
log.Error("File Copy Error: ", err)
return "", errors.New("File Failure")
return "", fmt.Errorf("File Failure")
}
return tempFile.Name(), nil
}
func GoodReadsMostRead(c Cadence) ([]SearchItem, error) {
body, err := getPage("https://www.goodreads.com/book/most_read?category=all&country=US&duration=" + string(c))
if err != nil {
return nil, err
}
return parseGoodReads(body)
}
func GetBookURL(id string, bookType BookType) (string, error) {
// Derive Info URL
var infoURL string
@ -180,212 +169,6 @@ func getPage(page string) (io.ReadCloser, error) {
return resp.Body, err
}
func parseLibGenFiction(body io.ReadCloser) ([]SearchItem, error) {
// Parse
defer body.Close()
doc, err := goquery.NewDocumentFromReader(body)
if err != nil {
return nil, err
}
// Normalize Results
var allEntries []SearchItem
doc.Find("table.catalog tbody > tr").Each(func(ix int, rawBook *goquery.Selection) {
// Parse File Details
fileItem := rawBook.Find("td:nth-child(5)")
fileDesc := fileItem.Text()
fileDescSplit := strings.Split(fileDesc, "/")
fileType := strings.ToLower(strings.TrimSpace(fileDescSplit[0]))
fileSize := strings.TrimSpace(fileDescSplit[1])
// Parse Upload Date
uploadedRaw, _ := fileItem.Attr("title")
uploadedDateRaw := strings.Split(uploadedRaw, "Uploaded at ")[1]
uploadDate, _ := time.Parse("2006-01-02 15:04:05", uploadedDateRaw)
// Parse MD5
editHref, _ := rawBook.Find("td:nth-child(7) a").Attr("href")
hrefArray := strings.Split(editHref, "/")
id := hrefArray[len(hrefArray)-1]
// Parse Other Details
title := rawBook.Find("td:nth-child(3) p a").Text()
author := rawBook.Find(".catalog_authors li a").Text()
language := rawBook.Find("td:nth-child(4)").Text()
series := rawBook.Find("td:nth-child(2)").Text()
item := SearchItem{
ID: id,
Title: title,
Author: author,
Series: series,
Language: language,
FileType: fileType,
FileSize: fileSize,
UploadDate: uploadDate.Format(time.RFC3339),
}
allEntries = append(allEntries, item)
})
// Return Results
return allEntries, nil
}
func parseLibGenNonFiction(body io.ReadCloser) ([]SearchItem, error) {
// Parse
defer body.Close()
doc, err := goquery.NewDocumentFromReader(body)
if err != nil {
return nil, err
}
// Normalize Results
var allEntries []SearchItem
doc.Find("table.c tbody > tr:nth-child(n + 2)").Each(func(ix int, rawBook *goquery.Selection) {
// Parse Type & Size
fileSize := strings.ToLower(strings.TrimSpace(rawBook.Find("td:nth-child(8)").Text()))
fileType := strings.ToLower(strings.TrimSpace(rawBook.Find("td:nth-child(9)").Text()))
// Parse MD5
titleRaw := rawBook.Find("td:nth-child(3) [id]")
editHref, _ := titleRaw.Attr("href")
hrefArray := strings.Split(editHref, "?md5=")
id := hrefArray[1]
// Parse Other Details
title := titleRaw.Text()
author := rawBook.Find("td:nth-child(2)").Text()
language := rawBook.Find("td:nth-child(7)").Text()
series := rawBook.Find("td:nth-child(3) [href*='column=series']").Text()
item := SearchItem{
ID: id,
Title: title,
Author: author,
Series: series,
Language: language,
FileType: fileType,
FileSize: fileSize,
}
allEntries = append(allEntries, item)
})
// Return Results
return allEntries, nil
}
func parseLibGenDownloadURL(body io.ReadCloser) (string, error) {
// Parse
defer body.Close()
doc, _ := goquery.NewDocumentFromReader(body)
// Return Download URL
// downloadURL, _ := doc.Find("#download [href*=cloudflare]").Attr("href")
downloadURL, exists := doc.Find("#download h2 a").Attr("href")
if exists == false {
return "", errors.New("Download URL not found")
}
return downloadURL, nil
}
func parseGoodReads(body io.ReadCloser) ([]SearchItem, error) {
// Parse
defer body.Close()
doc, err := goquery.NewDocumentFromReader(body)
if err != nil {
return nil, err
}
// Normalize Results
var allEntries []SearchItem
doc.Find("[itemtype=\"http://schema.org/Book\"]").Each(func(ix int, rawBook *goquery.Selection) {
title := rawBook.Find(".bookTitle span").Text()
author := rawBook.Find(".authorName span").Text()
item := SearchItem{
Title: title,
Author: author,
}
allEntries = append(allEntries, item)
})
// Return Results
return allEntries, nil
}
func parseAnnasArchiveDownloadURL(body io.ReadCloser) (string, error) {
// Parse
defer body.Close()
doc, _ := goquery.NewDocumentFromReader(body)
// Return Download URL
downloadURL, exists := doc.Find("body > table > tbody > tr > td > a").Attr("href")
if exists == false {
return "", errors.New("Download URL not found")
}
// Possible Funky URL
downloadURL = strings.ReplaceAll(downloadURL, "\\", "/")
return downloadURL, nil
}
func parseAnnasArchive(body io.ReadCloser) ([]SearchItem, error) {
// Parse
defer body.Close()
doc, err := goquery.NewDocumentFromReader(body)
if err != nil {
return nil, err
}
// Normalize Results
var allEntries []SearchItem
doc.Find("form > div.w-full > div.w-full > div > div.justify-center").Each(func(ix int, rawBook *goquery.Selection) {
// Parse Details
details := rawBook.Find("div:nth-child(2) > div:nth-child(1)").Text()
detailsSplit := strings.Split(details, ", ")
// Invalid Details
if len(detailsSplit) < 3 {
return
}
language := detailsSplit[0]
fileType := detailsSplit[1]
fileSize := detailsSplit[2]
// Get Title & Author
title := rawBook.Find("h3").Text()
author := rawBook.Find("div:nth-child(2) > div:nth-child(4)").Text()
// Parse MD5
itemHref, _ := rawBook.Find("a").Attr("href")
hrefArray := strings.Split(itemHref, "/")
id := hrefArray[len(hrefArray)-1]
item := SearchItem{
ID: id,
Title: title,
Author: author,
Language: language,
FileType: fileType,
FileSize: fileSize,
}
allEntries = append(allEntries, item)
})
// Return Results
return allEntries, nil
}
func downloadBook(bookURL string) (*http.Response, error) {
// Allow Insecure
client := &http.Client{Transport: &http.Transport{

View File

@ -23,7 +23,9 @@
</button>
</form>
</div>
<div class="flex flex-col-reverse text-black dark:text-white"
<!-- Required for iOS "Hover" Events (onclick) -->
<div onclick
class="flex flex-col-reverse text-black dark:text-white w-full overflow-scroll"
style="font-family: monospace">
{{ range $log := .Data }}
<span class="whitespace-nowrap hover:whitespace-pre">{{ $log }}</span>

View File

@ -10,10 +10,10 @@ import (
)
// Reimplemented KOReader Partial MD5 Calculation
func CalculatePartialMD5(filePath string) (string, error) {
func CalculatePartialMD5(filePath string) (*string, error) {
file, err := os.Open(filePath)
if err != nil {
return "", err
return nil, err
}
defer file.Close()
@ -41,7 +41,8 @@ func CalculatePartialMD5(filePath string) (string, error) {
}
allBytes := buf.Bytes()
return fmt.Sprintf("%x", md5.Sum(allBytes)), nil
fileHash := fmt.Sprintf("%x", md5.Sum(allBytes))
return &fileHash, nil
}
// Creates a token of n size
@ -53,3 +54,23 @@ func GenerateToken(n int) ([]byte, error) {
}
return b, nil
}
// Calculate MD5 of a file
func CalculateMD5(filePath string) (*string, error) {
file, err := os.Open(filePath)
if err != nil {
return nil, err
}
defer file.Close()
hash := md5.New()
_, err = io.Copy(hash, file)
if err != nil {
return nil, err
}
fileHash := fmt.Sprintf("%x", hash.Sum(nil))
return &fileHash, nil
}

View File

@ -1,12 +1,26 @@
package utils
import "testing"
import (
"github.com/stretchr/testify/assert"
"testing"
)
func TestCalculatePartialMD5(t *testing.T) {
partialMD5, err := CalculatePartialMD5("../_test_files/alice.epub")
assert := assert.New(t)
want := "386d1cb51fe4a72e5c9fdad5e059bad9"
if partialMD5 != want {
t.Fatalf(`Expected: %v, Got: %v, Error: %v`, want, partialMD5, err)
}
desiredPartialMD5 := "386d1cb51fe4a72e5c9fdad5e059bad9"
calculatedPartialMD5, err := CalculatePartialMD5("../_test_files/alice.epub")
assert.Nil(err, "error should be nil")
assert.Equal(desiredPartialMD5, *calculatedPartialMD5, "should be equal")
}
func TestCalculateMD5(t *testing.T) {
assert := assert.New(t)
desiredMD5 := "0f36c66155de34b281c4791654d0b1ce"
calculatedMD5, err := CalculateMD5("../_test_files/alice.epub")
assert.Nil(err, "error should be nil")
assert.Equal(desiredMD5, *calculatedMD5, "should be equal")
}