From 9b2b7bb6f0267737442b7d296907ebf13d81f946 Mon Sep 17 00:00:00 2001 From: Evan Reichard Date: Sat, 23 Sep 2023 14:14:57 -0400 Subject: [PATCH] [add] editing, deletion, metadata recording --- api/api.go | 5 +- api/app-routes.go | 237 ++++++++++++++++++++++++++----- api/ko-routes.go | 9 +- database/models.go | 17 +++ database/query.sql | 51 +++++-- database/query.sql.go | 154 +++++++++++++++++--- database/schema.sql | 25 ++++ metadata/metadata.go | 106 +++++++++----- sqlc.yaml | 45 ++++++ templates/document-edit.html | 166 ++++++++++++++++++++++ templates/document.html | 266 +++++++++++++++++++++++++++++------ templates/documents.html | 2 +- 12 files changed, 920 insertions(+), 163 deletions(-) create mode 100644 templates/document-edit.html diff --git a/api/api.go b/api/api.go index 78a773f..2627c94 100644 --- a/api/api.go +++ b/api/api.go @@ -25,7 +25,7 @@ type API struct { func NewApi(db *database.DBManager, c *config.Config) *API { api := &API{ - HTMLPolicy: bluemonday.StripTagsPolicy(), + HTMLPolicy: bluemonday.StrictPolicy(), Router: gin.Default(), Config: c, DB: db, @@ -97,6 +97,9 @@ func (api *API) registerWebAppRoutes() { api.Router.GET("/activity", api.authWebAppMiddleware, api.createAppResourcesRoute("activity")) api.Router.GET("/documents", api.authWebAppMiddleware, api.createAppResourcesRoute("documents")) api.Router.GET("/documents/:document", api.authWebAppMiddleware, api.createAppResourcesRoute("document")) + api.Router.DELETE("/documents/:document", api.authWebAppMiddleware, api.deleteDocument) + api.Router.POST("/documents/:document/edit", api.authWebAppMiddleware, api.createAppResourcesRoute("document-edit")) + api.Router.POST("/documents/:document/delete", api.authWebAppMiddleware, api.deleteDocument) api.Router.GET("/documents/:document/file", api.authWebAppMiddleware, api.downloadDocumentFile) api.Router.GET("/documents/:document/cover", api.authWebAppMiddleware, api.getDocumentCover) diff --git a/api/app-routes.go b/api/app-routes.go index 3670c97..48b00fd 100644 --- a/api/app-routes.go +++ b/api/app-routes.go @@ -2,17 +2,29 @@ package api import ( "fmt" + "mime/multipart" "net/http" "os" "path/filepath" + "strings" "time" + "github.com/gabriel-vasile/mimetype" "github.com/gin-gonic/gin" log "github.com/sirupsen/logrus" + "golang.org/x/exp/slices" "reichard.io/bbank/database" "reichard.io/bbank/metadata" ) +type requestDocumentEdit struct { + Title *string `form:"title"` + Author *string `form:"author"` + Description *string `form:"description"` + RemoveCover *string `form:"remove_cover"` + CoverFile *multipart.FileHeader `form:"cover"` +} + func baseResourceRoute(template string, args ...map[string]any) func(c *gin.Context) { variables := gin.H{"RouteName": template} if len(args) > 0 { @@ -84,6 +96,92 @@ func (api *API) createAppResourcesRoute(routeName string, args ...map[string]any } templateVars["Data"] = document + } else if routeName == "document-edit" { + var rDocID requestDocumentID + if err := c.ShouldBindUri(&rDocID); err != nil { + log.Error("[createAppResourcesRoute] Invalid URI Bind") + c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Request"}) + return + } + + var rDocEdit requestDocumentEdit + if err := c.ShouldBind(&rDocEdit); err != nil { + log.Error("[createAppResourcesRoute] Invalid Form Bind") + c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Request"}) + return + } + + // Validate Something Exists + if rDocEdit.Author == nil && + rDocEdit.Title == nil && + rDocEdit.Description == nil && + rDocEdit.CoverFile == nil && + rDocEdit.RemoveCover == nil { + log.Error("[createAppResourcesRoute] Missing Form Values") + c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Request"}) + return + } + + // Handle Cover + var coverFileName *string + if rDocEdit.RemoveCover != nil && *rDocEdit.RemoveCover == "on" { + s := "UNKNOWN" + coverFileName = &s + } else if rDocEdit.CoverFile != nil { + + // Validate Type & Derive Extension on MIME + uploadedFile, err := rDocEdit.CoverFile.Open() + if err != nil { + log.Error("[createAppResourcesRoute] File Error") + c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Request"}) + return + } + + fileMime, err := mimetype.DetectReader(uploadedFile) + if err != nil { + log.Error("[createAppResourcesRoute] MIME Error") + c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Request"}) + return + } + fileExtension := fileMime.Extension() + + // Validate Extension + if !slices.Contains([]string{".jpg", ".png"}, fileExtension) { + log.Error("[uploadDocumentFile] Invalid FileType: ", fileExtension) + c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Filetype"}) + return + } + + // Generate Storage Path + fileName := fmt.Sprintf("%s%s", rDocID.DocumentID, fileExtension) + safePath := filepath.Join(api.Config.DataPath, "covers", fileName) + + // Save + err = c.SaveUploadedFile(rDocEdit.CoverFile, safePath) + if err != nil { + log.Error("[createAppResourcesRoute] File Error") + c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Request"}) + return + } + + coverFileName = &fileName + } + + // Update Document + if _, err := api.DB.Queries.UpsertDocument(api.DB.Ctx, database.UpsertDocumentParams{ + ID: rDocID.DocumentID, + Title: api.sanitizeInput(rDocEdit.Title), + Author: api.sanitizeInput(rDocEdit.Author), + Description: api.sanitizeInput(rDocEdit.Description), + Coverfile: coverFileName, + }); err != nil { + log.Error("[createAppResourcesRoute] UpsertDocument DB Error:", err) + c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Request"}) + return + } + + c.Redirect(http.StatusFound, "./") + return } else if routeName == "activity" { activityFilter := database.GetActivityParams{ UserID: rUser.(string), @@ -164,19 +262,19 @@ func (api *API) getDocumentCover(c *gin.Context) { } // Handle Identified Document - if document.Olid != nil { - if *document.Olid == "UNKNOWN" { + if document.Coverfile != nil { + if *document.Coverfile == "UNKNOWN" { c.File("./assets/no-cover.jpg") return } // Derive Path - fileName := "." + filepath.Clean(fmt.Sprintf("/%s.jpg", *document.Olid)) - safePath := filepath.Join(api.Config.DataPath, "covers", fileName) + safePath := filepath.Join(api.Config.DataPath, "covers", *document.Coverfile) // Validate File Exists _, err = os.Stat(safePath) if err != nil { + log.Error("[getDocumentCover] File Should But Doesn't Exist:", err) c.File("./assets/no-cover.jpg") return } @@ -185,59 +283,126 @@ func (api *API) getDocumentCover(c *gin.Context) { return } - /* - This is a bit convoluted because we want to ensure we set the OLID to - UNKNOWN if there are any errors. This will ideally prevent us from - hitting the OpenLibrary API multiple times in the future. - */ + // --- Attempt Metadata --- - var coverID string = "UNKNOWN" - var coverFilePath string + var coverDir string = filepath.Join(api.Config.DataPath, "covers") + var coverFile string = "UNKNOWN" // Identify Documents & Save Covers - bookMetadata := metadata.MetadataInfo{ + metadataResults, err := metadata.GetMetadata(metadata.MetadataInfo{ Title: document.Title, Author: document.Author, - } - err = metadata.GetMetadata(&bookMetadata) - if err == nil && bookMetadata.GBID != nil { - // Derive & Sanitize File Name - fileName := "." + filepath.Clean(fmt.Sprintf("/%s.jpg", *bookMetadata.GBID)) + }) - // Generate Storage Path - coverFilePath = filepath.Join(api.Config.DataPath, "covers", fileName) + if err == nil && len(metadataResults) > 0 && metadataResults[0].GBID != nil { + firstResult := metadataResults[0] - err := metadata.SaveCover(*bookMetadata.GBID, coverFilePath) + // Save Cover + fileName, err := metadata.SaveCover(*firstResult.GBID, coverDir, document.ID) if err == nil { - coverID = *bookMetadata.GBID - log.Info("Title:", *bookMetadata.Title) - log.Info("Author:", *bookMetadata.Author) - log.Info("Description:", *bookMetadata.Description) - log.Info("IDs:", bookMetadata.ISBN) + coverFile = *fileName + } + + // Store First Metadata Result + if _, err = api.DB.Queries.AddMetadata(api.DB.Ctx, database.AddMetadataParams{ + DocumentID: document.ID, + Title: firstResult.Title, + Author: firstResult.Author, + Description: firstResult.Description, + Gbid: firstResult.GBID, + Olid: firstResult.OLID, + Isbn10: firstResult.ISBN10, + Isbn13: firstResult.ISBN13, + }); err != nil { + log.Error("[getDocumentCover] AddMetadata DB Error:", err) } } - // coverIDs, err := metadata.GetCoverOLIDs(document.Title, document.Author) - // if err == nil && len(coverIDs) > 0 { - // coverFilePath, err = metadata.DownloadAndSaveCover(coverIDs[0], api.Config.DataPath) - // if err == nil { - // coverID = coverIDs[0] - // } - // } - // Upsert Document if _, err = api.DB.Queries.UpsertDocument(api.DB.Ctx, database.UpsertDocumentParams{ - ID: document.ID, - Olid: &coverID, + ID: document.ID, + Coverfile: &coverFile, }); err != nil { log.Warn("[getDocumentCover] UpsertDocument DB Error:", err) } // Return Unknown Cover - if coverID == "UNKNOWN" { + if coverFile == "UNKNOWN" { c.File("./assets/no-cover.jpg") return } + coverFilePath := filepath.Join(coverDir, coverFile) c.File(coverFilePath) } + +// DELETE /api/documents/:document +func (api *API) deleteDocument(c *gin.Context) { + var rDocID requestDocumentID + if err := c.ShouldBindUri(&rDocID); err != nil { + log.Error("[deleteDocument] Invalid URI Bind") + c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Request"}) + return + } + changed, err := api.DB.Queries.DeleteDocument(api.DB.Ctx, rDocID.DocumentID) + if err != nil { + log.Error("[deleteDocument] DeleteDocument DB Error") + c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Request"}) + return + } + if changed == 0 { + log.Error("[deleteDocument] DeleteDocument DB Error") + c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Unknown Document"}) + return + } + + c.Redirect(http.StatusFound, "../") +} + +// POST /api/documents/:document/identify +func (api *API) identifyDocument(c *gin.Context) { + var rDocID requestDocumentID + if err := c.ShouldBindUri(&rDocID); err != nil { + log.Error("[identifyDocument] Invalid URI Bind") + c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Request"}) + return + } + + isbn := strings.TrimSpace(c.PostForm("ISBN")) + if isbn == "" { + log.Error("[identifyDocument] Invalid Form") + c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Request"}) + return + } + + metadataResults, err := metadata.GetMetadata(metadata.MetadataInfo{ + ISBN10: &isbn, + ISBN13: &isbn, + }) + if err != nil || len(metadataResults) == 0 { + log.Error("[identifyDocument] Metadata Error") + c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Metadata Error"}) + return + } + + // TODO + firstResult := metadataResults[0] + + if firstResult.Title != nil { + log.Info("Title:", *firstResult.Title) + } + if firstResult.Author != nil { + log.Info("Author:", *firstResult.Author) + } + if firstResult.Description != nil { + log.Info("Description:", *firstResult.Description) + } + if firstResult.ISBN10 != nil { + log.Info("ISBN 10:", *firstResult.ISBN10) + } + if firstResult.ISBN13 != nil { + log.Info("ISBN 13:", *firstResult.ISBN13) + } + + c.Redirect(http.StatusFound, "/") +} diff --git a/api/ko-routes.go b/api/ko-routes.go index df16738..115e446 100644 --- a/api/ko-routes.go +++ b/api/ko-routes.go @@ -5,6 +5,7 @@ import ( "database/sql" "encoding/json" "fmt" + "html" "io" "net/http" "os" @@ -72,8 +73,6 @@ type requestDocumentID struct { DocumentID string `uri:"document" binding:"required"` } -var allowedExtensions []string = []string{".epub", ".html"} - func (api *API) authorizeUser(c *gin.Context) { c.JSON(200, gin.H{ "authorized": "OK", @@ -485,7 +484,7 @@ func (api *API) uploadDocumentFile(c *gin.Context) { fileMime, err := mimetype.DetectReader(uploadedFile) fileExtension := fileMime.Extension() - if !slices.Contains(allowedExtensions, fileExtension) { + if !slices.Contains([]string{".epub", ".html"}, fileExtension) { log.Error("[uploadDocumentFile] Invalid FileType:", fileExtension) c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Filetype"}) return @@ -609,12 +608,12 @@ func (api *API) sanitizeInput(val any) *string { switch v := val.(type) { case *string: if v != nil { - newString := api.HTMLPolicy.Sanitize(string(*v)) + newString := html.UnescapeString(api.HTMLPolicy.Sanitize(string(*v))) return &newString } case string: if v != "" { - newString := api.HTMLPolicy.Sanitize(string(v)) + newString := html.UnescapeString(api.HTMLPolicy.Sanitize(string(v))) return &newString } } diff --git a/database/models.go b/database/models.go index 19eb8a3..dc6d919 100644 --- a/database/models.go +++ b/database/models.go @@ -32,13 +32,17 @@ type Document struct { ID string `json:"id"` Md5 *string `json:"md5"` Filepath *string `json:"filepath"` + Coverfile *string `json:"coverfile"` Title *string `json:"title"` Author *string `json:"author"` Series *string `json:"series"` SeriesIndex *int64 `json:"series_index"` Lang *string `json:"lang"` Description *string `json:"description"` + Gbid *string `json:"gbid"` Olid *string `json:"-"` + Isbn10 *string `json:"isbn10"` + Isbn13 *string `json:"isbn13"` Synced bool `json:"-"` Deleted bool `json:"-"` UpdatedAt time.Time `json:"updated_at"` @@ -62,6 +66,19 @@ type DocumentProgress struct { CreatedAt time.Time `json:"created_at"` } +type Metadatum struct { + ID int64 `json:"id"` + DocumentID string `json:"document_id"` + Title *string `json:"title"` + Author *string `json:"author"` + Description *string `json:"description"` + Gbid *string `json:"gbid"` + Olid *string `json:"olid"` + Isbn10 *string `json:"isbn10"` + Isbn13 *string `json:"isbn13"` + CreatedAt time.Time `json:"created_at"` +} + type RescaledActivity struct { DocumentID string `json:"document_id"` DeviceID string `json:"device_id"` diff --git a/database/query.sql b/database/query.sql index 3bd8ad9..e60c9f8 100644 --- a/database/query.sql +++ b/database/query.sql @@ -1,3 +1,17 @@ +-- name: AddMetadata :one +INSERT INTO metadata ( + document_id, + title, + author, + description, + gbid, + olid, + isbn10, + isbn13 +) +VALUES (?, ?, ?, ?, ?, ?, ?, ?) +RETURNING *; + -- name: CreateUser :execrows INSERT INTO users (id, pass) VALUES (?, ?) @@ -12,26 +26,34 @@ INSERT INTO documents ( id, md5, filepath, + coverfile, title, author, series, series_index, lang, description, - olid + olid, + gbid, + isbn10, + isbn13 ) -VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT DO UPDATE SET md5 = COALESCE(excluded.md5, md5), filepath = COALESCE(excluded.filepath, filepath), + coverfile = COALESCE(excluded.coverfile, coverfile), title = COALESCE(excluded.title, title), author = COALESCE(excluded.author, author), series = COALESCE(excluded.series, series), series_index = COALESCE(excluded.series_index, series_index), lang = COALESCE(excluded.lang, lang), description = COALESCE(excluded.description, description), - olid = COALESCE(excluded.olid, olid) + olid = COALESCE(excluded.olid, olid), + gbid = COALESCE(excluded.gbid, gbid), + isbn10 = COALESCE(excluded.isbn10, isbn10), + isbn13 = COALESCE(excluded.isbn13, isbn13) RETURNING *; -- name: DeleteDocument :execrows @@ -222,6 +244,7 @@ SELECT FROM documents LEFT JOIN true_progress ON true_progress.document_id = documents.id LEFT JOIN users ON users.id = $user_id +WHERE documents.deleted == false ORDER BY true_progress.last_read DESC, documents.created_at DESC LIMIT $limit OFFSET $offset; @@ -308,11 +331,11 @@ FROM document_days; -- name: GetUserWindowStreaks :one WITH document_windows AS ( SELECT - CASE - WHEN ?2 = "WEEK" THEN DATE(start_time, time_offset, 'weekday 0', '-7 day') - WHEN ?2 = "DAY" THEN DATE(start_time, time_offset) - END AS read_window, - time_offset + CASE + WHEN ?2 = "WEEK" THEN DATE(start_time, time_offset, 'weekday 0', '-7 day') + WHEN ?2 = "DAY" THEN DATE(start_time, time_offset) + END AS read_window, + time_offset FROM activity JOIN users ON users.id = activity.user_id WHERE user_id = $user_id @@ -332,14 +355,14 @@ streaks AS ( count(*) AS streak, MIN(read_window) AS start_date, MAX(read_window) AS end_date, - time_offset + time_offset FROM partitions GROUP BY - CASE - WHEN ?2 = "DAY" THEN DATE(read_window, '+' || seqnum || ' day') - WHEN ?2 = "WEEK" THEN DATE(read_window, '+' || (seqnum * 7) || ' day') - END, - time_offset + CASE + WHEN ?2 = "DAY" THEN DATE(read_window, '+' || seqnum || ' day') + WHEN ?2 = "WEEK" THEN DATE(read_window, '+' || (seqnum * 7) || ' day') + END, + time_offset ORDER BY end_date DESC ), max_streak AS ( diff --git a/database/query.sql.go b/database/query.sql.go index 1795fdf..54ebaea 100644 --- a/database/query.sql.go +++ b/database/query.sql.go @@ -61,6 +61,59 @@ func (q *Queries) AddActivity(ctx context.Context, arg AddActivityParams) (Activ return i, err } +const addMetadata = `-- name: AddMetadata :one +INSERT INTO metadata ( + document_id, + title, + author, + description, + gbid, + olid, + isbn10, + isbn13 +) +VALUES (?, ?, ?, ?, ?, ?, ?, ?) +RETURNING id, document_id, title, author, description, gbid, olid, isbn10, isbn13, created_at +` + +type AddMetadataParams struct { + DocumentID string `json:"document_id"` + Title *string `json:"title"` + Author *string `json:"author"` + Description *string `json:"description"` + Gbid *string `json:"gbid"` + Olid *string `json:"olid"` + Isbn10 *string `json:"isbn10"` + Isbn13 *string `json:"isbn13"` +} + +func (q *Queries) AddMetadata(ctx context.Context, arg AddMetadataParams) (Metadatum, error) { + row := q.db.QueryRowContext(ctx, addMetadata, + arg.DocumentID, + arg.Title, + arg.Author, + arg.Description, + arg.Gbid, + arg.Olid, + arg.Isbn10, + arg.Isbn13, + ) + var i Metadatum + err := row.Scan( + &i.ID, + &i.DocumentID, + &i.Title, + &i.Author, + &i.Description, + &i.Gbid, + &i.Olid, + &i.Isbn10, + &i.Isbn13, + &i.CreatedAt, + ) + return i, err +} + const createUser = `-- name: CreateUser :execrows INSERT INTO users (id, pass) VALUES (?, ?) @@ -366,7 +419,7 @@ func (q *Queries) GetDevices(ctx context.Context, arg GetDevicesParams) ([]Devic } const getDocument = `-- name: GetDocument :one -SELECT id, md5, filepath, title, author, series, series_index, lang, description, olid, synced, deleted, updated_at, created_at FROM documents +SELECT id, md5, filepath, coverfile, title, author, series, series_index, lang, description, gbid, olid, isbn10, isbn13, synced, deleted, updated_at, created_at FROM documents WHERE id = ?1 LIMIT 1 ` @@ -377,13 +430,17 @@ func (q *Queries) GetDocument(ctx context.Context, documentID string) (Document, &i.ID, &i.Md5, &i.Filepath, + &i.Coverfile, &i.Title, &i.Author, &i.Series, &i.SeriesIndex, &i.Lang, &i.Description, + &i.Gbid, &i.Olid, + &i.Isbn10, + &i.Isbn13, &i.Synced, &i.Deleted, &i.UpdatedAt, @@ -501,7 +558,7 @@ WITH true_progress AS ( LIMIT 1 ) SELECT - documents.id, documents.md5, documents.filepath, documents.title, documents.author, documents.series, documents.series_index, documents.lang, documents.description, documents.olid, documents.synced, documents.deleted, documents.updated_at, documents.created_at, + documents.id, documents.md5, documents.filepath, documents.coverfile, documents.title, documents.author, documents.series, documents.series_index, documents.lang, documents.description, documents.gbid, documents.olid, documents.isbn10, documents.isbn13, documents.synced, documents.deleted, documents.updated_at, documents.created_at, CAST(IFNULL(current_page, 0) AS INTEGER) AS current_page, CAST(IFNULL(total_pages, 0) AS INTEGER) AS total_pages, @@ -531,13 +588,17 @@ type GetDocumentWithStatsRow struct { ID string `json:"id"` Md5 *string `json:"md5"` Filepath *string `json:"filepath"` + Coverfile *string `json:"coverfile"` Title *string `json:"title"` Author *string `json:"author"` Series *string `json:"series"` SeriesIndex *int64 `json:"series_index"` Lang *string `json:"lang"` Description *string `json:"description"` + Gbid *string `json:"gbid"` Olid *string `json:"-"` + Isbn10 *string `json:"isbn10"` + Isbn13 *string `json:"isbn13"` Synced bool `json:"-"` Deleted bool `json:"-"` UpdatedAt time.Time `json:"updated_at"` @@ -556,13 +617,17 @@ func (q *Queries) GetDocumentWithStats(ctx context.Context, arg GetDocumentWithS &i.ID, &i.Md5, &i.Filepath, + &i.Coverfile, &i.Title, &i.Author, &i.Series, &i.SeriesIndex, &i.Lang, &i.Description, + &i.Gbid, &i.Olid, + &i.Isbn10, + &i.Isbn13, &i.Synced, &i.Deleted, &i.UpdatedAt, @@ -577,7 +642,7 @@ func (q *Queries) GetDocumentWithStats(ctx context.Context, arg GetDocumentWithS } const getDocuments = `-- name: GetDocuments :many -SELECT id, md5, filepath, title, author, series, series_index, lang, description, olid, synced, deleted, updated_at, created_at FROM documents +SELECT id, md5, filepath, coverfile, title, author, series, series_index, lang, description, gbid, olid, isbn10, isbn13, synced, deleted, updated_at, created_at FROM documents ORDER BY created_at DESC LIMIT ?2 OFFSET ?1 @@ -601,13 +666,17 @@ func (q *Queries) GetDocuments(ctx context.Context, arg GetDocumentsParams) ([]D &i.ID, &i.Md5, &i.Filepath, + &i.Coverfile, &i.Title, &i.Author, &i.Series, &i.SeriesIndex, &i.Lang, &i.Description, + &i.Gbid, &i.Olid, + &i.Isbn10, + &i.Isbn13, &i.Synced, &i.Deleted, &i.UpdatedAt, @@ -641,7 +710,7 @@ WITH true_progress AS ( HAVING MAX(start_time) ) SELECT - documents.id, documents.md5, documents.filepath, documents.title, documents.author, documents.series, documents.series_index, documents.lang, documents.description, documents.olid, documents.synced, documents.deleted, documents.updated_at, documents.created_at, + documents.id, documents.md5, documents.filepath, documents.coverfile, documents.title, documents.author, documents.series, documents.series_index, documents.lang, documents.description, documents.gbid, documents.olid, documents.isbn10, documents.isbn13, documents.synced, documents.deleted, documents.updated_at, documents.created_at, CAST(IFNULL(current_page, 0) AS INTEGER) AS current_page, CAST(IFNULL(total_pages, 0) AS INTEGER) AS total_pages, @@ -657,6 +726,7 @@ SELECT FROM documents LEFT JOIN true_progress ON true_progress.document_id = documents.id LEFT JOIN users ON users.id = ?1 +WHERE documents.deleted == false ORDER BY true_progress.last_read DESC, documents.created_at DESC LIMIT ?3 OFFSET ?2 @@ -672,13 +742,17 @@ type GetDocumentsWithStatsRow struct { ID string `json:"id"` Md5 *string `json:"md5"` Filepath *string `json:"filepath"` + Coverfile *string `json:"coverfile"` Title *string `json:"title"` Author *string `json:"author"` Series *string `json:"series"` SeriesIndex *int64 `json:"series_index"` Lang *string `json:"lang"` Description *string `json:"description"` + Gbid *string `json:"gbid"` Olid *string `json:"-"` + Isbn10 *string `json:"isbn10"` + Isbn13 *string `json:"isbn13"` Synced bool `json:"-"` Deleted bool `json:"-"` UpdatedAt time.Time `json:"updated_at"` @@ -703,13 +777,17 @@ func (q *Queries) GetDocumentsWithStats(ctx context.Context, arg GetDocumentsWit &i.ID, &i.Md5, &i.Filepath, + &i.Coverfile, &i.Title, &i.Author, &i.Series, &i.SeriesIndex, &i.Lang, &i.Description, + &i.Gbid, &i.Olid, + &i.Isbn10, + &i.Isbn13, &i.Synced, &i.Deleted, &i.UpdatedAt, @@ -754,7 +832,7 @@ func (q *Queries) GetLastActivity(ctx context.Context, arg GetLastActivityParams } const getMissingDocuments = `-- name: GetMissingDocuments :many -SELECT documents.id, documents.md5, documents.filepath, documents.title, documents.author, documents.series, documents.series_index, documents.lang, documents.description, documents.olid, documents.synced, documents.deleted, documents.updated_at, documents.created_at FROM documents +SELECT documents.id, documents.md5, documents.filepath, documents.coverfile, documents.title, documents.author, documents.series, documents.series_index, documents.lang, documents.description, documents.gbid, documents.olid, documents.isbn10, documents.isbn13, documents.synced, documents.deleted, documents.updated_at, documents.created_at FROM documents WHERE documents.filepath IS NOT NULL AND documents.deleted = false @@ -784,13 +862,17 @@ func (q *Queries) GetMissingDocuments(ctx context.Context, documentIds []string) &i.ID, &i.Md5, &i.Filepath, + &i.Coverfile, &i.Title, &i.Author, &i.Series, &i.SeriesIndex, &i.Lang, &i.Description, + &i.Gbid, &i.Olid, + &i.Isbn10, + &i.Isbn13, &i.Synced, &i.Deleted, &i.UpdatedAt, @@ -875,11 +957,11 @@ func (q *Queries) GetUser(ctx context.Context, userID string) (User, error) { const getUserWindowStreaks = `-- name: GetUserWindowStreaks :one WITH document_windows AS ( SELECT - CASE - WHEN ?2 = "WEEK" THEN DATE(start_time, time_offset, 'weekday 0', '-7 day') - WHEN ?2 = "DAY" THEN DATE(start_time, time_offset) - END AS read_window, - time_offset + CASE + WHEN ?2 = "WEEK" THEN DATE(start_time, time_offset, 'weekday 0', '-7 day') + WHEN ?2 = "DAY" THEN DATE(start_time, time_offset) + END AS read_window, + time_offset FROM activity JOIN users ON users.id = activity.user_id WHERE user_id = ?1 @@ -899,14 +981,14 @@ streaks AS ( count(*) AS streak, MIN(read_window) AS start_date, MAX(read_window) AS end_date, - time_offset + time_offset FROM partitions GROUP BY - CASE - WHEN ?2 = "DAY" THEN DATE(read_window, '+' || seqnum || ' day') - WHEN ?2 = "WEEK" THEN DATE(read_window, '+' || (seqnum * 7) || ' day') - END, - time_offset + CASE + WHEN ?2 = "DAY" THEN DATE(read_window, '+' || seqnum || ' day') + WHEN ?2 = "WEEK" THEN DATE(read_window, '+' || (seqnum * 7) || ' day') + END, + time_offset ORDER BY end_date DESC ), max_streak AS ( @@ -1073,7 +1155,7 @@ UPDATE documents SET deleted = ?1 WHERE id = ?2 -RETURNING id, md5, filepath, title, author, series, series_index, lang, description, olid, synced, deleted, updated_at, created_at +RETURNING id, md5, filepath, coverfile, title, author, series, series_index, lang, description, gbid, olid, isbn10, isbn13, synced, deleted, updated_at, created_at ` type UpdateDocumentDeletedParams struct { @@ -1088,13 +1170,17 @@ func (q *Queries) UpdateDocumentDeleted(ctx context.Context, arg UpdateDocumentD &i.ID, &i.Md5, &i.Filepath, + &i.Coverfile, &i.Title, &i.Author, &i.Series, &i.SeriesIndex, &i.Lang, &i.Description, + &i.Gbid, &i.Olid, + &i.Isbn10, + &i.Isbn13, &i.Synced, &i.Deleted, &i.UpdatedAt, @@ -1108,7 +1194,7 @@ UPDATE documents SET synced = ?1 WHERE id = ?2 -RETURNING id, md5, filepath, title, author, series, series_index, lang, description, olid, synced, deleted, updated_at, created_at +RETURNING id, md5, filepath, coverfile, title, author, series, series_index, lang, description, gbid, olid, isbn10, isbn13, synced, deleted, updated_at, created_at ` type UpdateDocumentSyncParams struct { @@ -1123,13 +1209,17 @@ func (q *Queries) UpdateDocumentSync(ctx context.Context, arg UpdateDocumentSync &i.ID, &i.Md5, &i.Filepath, + &i.Coverfile, &i.Title, &i.Author, &i.Series, &i.SeriesIndex, &i.Lang, &i.Description, + &i.Gbid, &i.Olid, + &i.Isbn10, + &i.Isbn13, &i.Synced, &i.Deleted, &i.UpdatedAt, @@ -1211,33 +1301,42 @@ INSERT INTO documents ( id, md5, filepath, + coverfile, title, author, series, series_index, lang, description, - olid + olid, + gbid, + isbn10, + isbn13 ) -VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT DO UPDATE SET md5 = COALESCE(excluded.md5, md5), filepath = COALESCE(excluded.filepath, filepath), + coverfile = COALESCE(excluded.coverfile, coverfile), title = COALESCE(excluded.title, title), author = COALESCE(excluded.author, author), series = COALESCE(excluded.series, series), series_index = COALESCE(excluded.series_index, series_index), lang = COALESCE(excluded.lang, lang), description = COALESCE(excluded.description, description), - olid = COALESCE(excluded.olid, olid) -RETURNING id, md5, filepath, title, author, series, series_index, lang, description, olid, synced, deleted, updated_at, created_at + olid = COALESCE(excluded.olid, olid), + gbid = COALESCE(excluded.gbid, gbid), + isbn10 = COALESCE(excluded.isbn10, isbn10), + isbn13 = COALESCE(excluded.isbn13, isbn13) +RETURNING id, md5, filepath, coverfile, title, author, series, series_index, lang, description, gbid, olid, isbn10, isbn13, synced, deleted, updated_at, created_at ` type UpsertDocumentParams struct { ID string `json:"id"` Md5 *string `json:"md5"` Filepath *string `json:"filepath"` + Coverfile *string `json:"coverfile"` Title *string `json:"title"` Author *string `json:"author"` Series *string `json:"series"` @@ -1245,6 +1344,9 @@ type UpsertDocumentParams struct { Lang *string `json:"lang"` Description *string `json:"description"` Olid *string `json:"-"` + Gbid *string `json:"gbid"` + Isbn10 *string `json:"isbn10"` + Isbn13 *string `json:"isbn13"` } func (q *Queries) UpsertDocument(ctx context.Context, arg UpsertDocumentParams) (Document, error) { @@ -1252,6 +1354,7 @@ func (q *Queries) UpsertDocument(ctx context.Context, arg UpsertDocumentParams) arg.ID, arg.Md5, arg.Filepath, + arg.Coverfile, arg.Title, arg.Author, arg.Series, @@ -1259,19 +1362,26 @@ func (q *Queries) UpsertDocument(ctx context.Context, arg UpsertDocumentParams) arg.Lang, arg.Description, arg.Olid, + arg.Gbid, + arg.Isbn10, + arg.Isbn13, ) var i Document err := row.Scan( &i.ID, &i.Md5, &i.Filepath, + &i.Coverfile, &i.Title, &i.Author, &i.Series, &i.SeriesIndex, &i.Lang, &i.Description, + &i.Gbid, &i.Olid, + &i.Isbn10, + &i.Isbn13, &i.Synced, &i.Deleted, &i.UpdatedAt, diff --git a/database/schema.sql b/database/schema.sql index 84679fa..db62b06 100644 --- a/database/schema.sql +++ b/database/schema.sql @@ -18,13 +18,19 @@ CREATE TABLE IF NOT EXISTS documents ( md5 TEXT, filepath TEXT, + coverfile TEXT, title TEXT, author TEXT, series TEXT, series_index INTEGER, lang TEXT, description TEXT, + + gbid TEXT, olid TEXT, + isbn10 TEXT, + isbn13 TEXT, + synced BOOLEAN NOT NULL DEFAULT 0 CHECK (synced IN (0, 1)), deleted BOOLEAN NOT NULL DEFAULT 0 CHECK (deleted IN (0, 1)), @@ -32,6 +38,25 @@ CREATE TABLE IF NOT EXISTS documents ( created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ); +-- Metadata +CREATE TABLE IF NOT EXISTS metadata ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + + document_id TEXT NOT NULL, + + title TEXT, + author TEXT, + description TEXT, + gbid TEXT, + olid TEXT, + isbn10 TEXT, + isbn13 TEXT, + + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + + FOREIGN KEY (document_id) REFERENCES documents (id) +); + -- Devices CREATE TABLE IF NOT EXISTS devices ( id TEXT NOT NULL PRIMARY KEY, diff --git a/metadata/metadata.go b/metadata/metadata.go index e6f8b68..e26b8f3 100644 --- a/metadata/metadata.go +++ b/metadata/metadata.go @@ -8,6 +8,7 @@ import ( "net/http" "net/url" "os" + "path/filepath" log "github.com/sirupsen/logrus" ) @@ -17,7 +18,9 @@ type MetadataInfo struct { Author *string Description *string GBID *string - ISBN []*string + OLID *string + ISBN10 *string + ISBN13 *string } type gBooksIdentifiers struct { @@ -46,73 +49,99 @@ const GBOOKS_QUERY_URL string = "https://www.googleapis.com/books/v1/volumes?q=% const GBOOKS_GBID_INFO_URL string = "https://www.googleapis.com/books/v1/volumes/%s" const GBOOKS_GBID_COVER_URL string = "https://books.google.com/books/content/images/frontcover/%s?fife=w480-h690" -func GetMetadata(data *MetadataInfo) error { - var queryResult *gBooksQueryItem - if data.GBID != nil { +func GetMetadata(metadataSearch MetadataInfo) ([]MetadataInfo, error) { + var queryResults []gBooksQueryItem + if metadataSearch.GBID != nil { // Use GBID - resp, err := performGBIDRequest(*data.GBID) + resp, err := performGBIDRequest(*metadataSearch.GBID) if err != nil { - return err + return nil, err } - queryResult = resp - } else if len(data.ISBN) > 0 { - searchQuery := "isbn:" + *data.ISBN[0] + queryResults = []gBooksQueryItem{*resp} + } else if metadataSearch.ISBN13 != nil { + searchQuery := "isbn:" + *metadataSearch.ISBN13 resp, err := performSearchRequest(searchQuery) if err != nil { - return err + return nil, err } - queryResult = &resp.Items[0] - } else if data.Title != nil && data.Author != nil { - searchQuery := url.QueryEscape(fmt.Sprintf("%s %s", *data.Title, *data.Author)) + queryResults = resp.Items + } else if metadataSearch.ISBN10 != nil { + searchQuery := "isbn:" + *metadataSearch.ISBN10 resp, err := performSearchRequest(searchQuery) if err != nil { - return err + return nil, err } - queryResult = &resp.Items[0] + queryResults = resp.Items + } else if metadataSearch.Title != nil && metadataSearch.Author != nil { + searchQuery := url.QueryEscape(fmt.Sprintf("%s %s", *metadataSearch.Title, *metadataSearch.Author)) + resp, err := performSearchRequest(searchQuery) + if err != nil { + return nil, err + } + queryResults = resp.Items } else { - return errors.New("Invalid Data") + return nil, errors.New("Invalid Data") } - // Merge Data - data.GBID = &queryResult.ID - data.Description = &queryResult.Info.Description - data.Title = &queryResult.Info.Title - if len(queryResult.Info.Authors) > 0 { - data.Author = &queryResult.Info.Authors[0] - } - for _, item := range queryResult.Info.Identifiers { - if item.Type == "ISBN_10" || item.Type == "ISBN_13" { - data.ISBN = append(data.ISBN, &item.Identifier) + // Normalize Data + allMetadata := []MetadataInfo{} + + for i := range queryResults { + // Account for range value changing every iteration + item := queryResults[i] + itemResult := MetadataInfo{ + GBID: &item.ID, + Title: &item.Info.Title, + Description: &item.Info.Description, } + if len(item.Info.Authors) > 0 { + itemResult.Author = &item.Info.Authors[0] + } + for _, item := range item.Info.Identifiers { + if itemResult.ISBN10 != nil && itemResult.ISBN13 != nil { + break + } else if itemResult.ISBN10 == nil && item.Type == "ISBN_10" { + itemResult.ISBN10 = &item.Identifier + } else if itemResult.ISBN13 == nil && item.Type == "ISBN_13" { + itemResult.ISBN13 = &item.Identifier + } + } + + allMetadata = append(allMetadata, itemResult) } - return nil + return allMetadata, nil } -func SaveCover(id string, safePath string) error { +func SaveCover(gbid string, coverDir string, documentID string) (*string, error) { + + // Google Books -> JPG + coverFile := "." + filepath.Clean(fmt.Sprintf("/%s.jpg", documentID)) + coverFilePath := filepath.Join(coverDir, coverFile) + // Validate File Doesn't Exists - _, err := os.Stat(safePath) + _, err := os.Stat(coverFilePath) if err == nil { log.Warn("[SaveCover] File Alreads Exists") - return nil + return &coverFile, nil } // Create File - out, err := os.Create(safePath) + out, err := os.Create(coverFilePath) if err != nil { log.Error("[SaveCover] File Create Error") - return errors.New("File Failure") + return nil, errors.New("File Failure") } defer out.Close() // Download File log.Info("[SaveCover] Downloading Cover") - coverURL := fmt.Sprintf(GBOOKS_GBID_COVER_URL, id) + coverURL := fmt.Sprintf(GBOOKS_GBID_COVER_URL, gbid) resp, err := http.Get(coverURL) if err != nil { log.Error("[SaveCover] Cover URL API Failure") - return errors.New("API Failure") + return nil, errors.New("API Failure") } defer resp.Body.Close() @@ -121,20 +150,19 @@ func SaveCover(id string, safePath string) error { _, err = io.Copy(out, resp.Body) if err != nil { log.Error("[SaveCover] File Copy Error") - return errors.New("File Failure") + return nil, errors.New("File Failure") } // Return FilePath - return nil + return &coverFile, nil } func performSearchRequest(searchQuery string) (*gBooksQueryResponse, error) { apiQuery := fmt.Sprintf(GBOOKS_QUERY_URL, searchQuery) - - log.Info("[performSearchRequest] Acquiring CoverID") + log.Info("[performSearchRequest] Acquiring Metadata: ", apiQuery) resp, err := http.Get(apiQuery) if err != nil { - log.Error("[performSearchRequest] Cover URL API Failure") + log.Error("[performSearchRequest] Google Books Query URL API Failure") return nil, errors.New("API Failure") } diff --git a/sqlc.yaml b/sqlc.yaml index 6f25df0..8d4c7fb 100644 --- a/sqlc.yaml +++ b/sqlc.yaml @@ -18,6 +18,10 @@ sql: go_type: type: "string" pointer: true + - column: "documents.coverfile" + go_type: + type: "string" + pointer: true - column: "documents.title" go_type: type: "string" @@ -46,6 +50,47 @@ sql: go_type: type: "string" pointer: true + - column: "documents.gbid" + go_type: + type: "string" + pointer: true + - column: "documents.isbn10" + go_type: + type: "string" + pointer: true + - column: "documents.isbn13" + go_type: + type: "string" + pointer: true + + - column: "metadata.title" + go_type: + type: "string" + pointer: true + - column: "metadata.author" + go_type: + type: "string" + pointer: true + - column: "metadata.description" + go_type: + type: "string" + pointer: true + - column: "metadata.gbid" + go_type: + type: "string" + pointer: true + - column: "metadata.olid" + go_type: + type: "string" + pointer: true + - column: "metadata.isbn10" + go_type: + type: "string" + pointer: true + - column: "metadata.isbn13" + go_type: + type: "string" + pointer: true # Do not generate JSON - column: "documents.synced" diff --git a/templates/document-edit.html b/templates/document-edit.html new file mode 100644 index 0000000..68d9027 --- /dev/null +++ b/templates/document-edit.html @@ -0,0 +1,166 @@ +{{template "base.html" .}} + +{{define "title"}}Documents{{end}} + +{{define "header"}} +Documents +{{end}} + +{{define "content"}} +
+
+ +
+
+ + +
+

Are you sure?

+ +
+
+ + + + + + +
+ + +
+ +
+ +
+
+ {{ if .Data.Filepath }} + + + + + + {{ else }} + + + + {{ end }} +
+
+
+
+

Title

+ +

+ {{ or .Data.Title "N/A" }} +

+
+
+

Author

+

+ {{ or .Data.Author "N/A" }} +

+
+
+

Progress

+

+ {{ .Data.CurrentPage }} / {{ .Data.TotalPages }} ({{ .Data.Percentage }}%) +

+
+
+

Time Read

+

+ {{ .Data.TotalTimeMinutes }} Minutes +

+
+
+

Description

+

+ {{ or .Data.Description "N/A" }} +

+
+ +{{end}} diff --git a/templates/document.html b/templates/document.html index d90ac70..c10d7cb 100644 --- a/templates/document.html +++ b/templates/document.html @@ -7,10 +7,37 @@ {{end}} {{define "content"}} -
-
- -
+
+
+ +
+ +
+
+ + +
+
+ + +
+ +
-
-

Are you sure?

- -
+
+
+

Are you sure?

+ +
+
-
{{ if .Data.Filepath }} @@ -116,44 +156,180 @@
-
-

Title

- -

- {{ or .Data.Title "N/A" }} -

+
+
+

Title

+ + +
+
+ + +
+
+
+

+ {{ or .Data.Title "N/A" }} +

+
+
+
+

Author

+ + + +
+
+ + +
+
+
+

+ {{ or .Data.Author "N/A" }} +

-

Author

-

- {{ or .Data.Author "N/A" }} +

Time Read

+

+ {{ .Data.TotalTimeMinutes }} Minutes

Progress

-

+

{{ .Data.CurrentPage }} / {{ .Data.TotalPages }} ({{ .Data.Percentage }}%)

+ +
+ +
+
+

Description

+ +
+
+
+ +
+ +
+ + +
+
+

{{ or .Data.Description "N/A" }}

-

Description

-

- {{ or .Data.Description "N/A" }} -