From 2a101c6ceeb529623ce7f27ba8217ae11f4dd845 Mon Sep 17 00:00:00 2001 From: Evan Reichard Date: Sun, 1 Oct 2023 19:17:22 -0400 Subject: [PATCH] [new] count words & stats, [new] refactor metadata, [new] human readable time --- api/api.go | 1 + api/app-routes.go | 106 ++++++++++--- database/models.go | 1 + database/query.sql | 33 +++- database/query.sql.go | 73 ++++++--- database/schema.sql | 1 + metadata/epub.go | 330 +++++++++++++++++++++++++++++++++++++++ metadata/gbooks.go | 200 ++++++++++++++++++++++++ metadata/metadata.go | 219 +++++--------------------- sqlc.yaml | 4 + templates/document.html | 56 ++++++- templates/documents.html | 2 +- utils/utils.go | 29 ++++ 13 files changed, 816 insertions(+), 239 deletions(-) create mode 100644 metadata/epub.go create mode 100644 metadata/gbooks.go diff --git a/api/api.go b/api/api.go index f69dfa0..9d6f651 100644 --- a/api/api.go +++ b/api/api.go @@ -76,6 +76,7 @@ func (api *API) registerWebAppRoutes() { helperFuncs := template.FuncMap{ "GetSVGGraphData": graph.GetSVGGraphData, "GetUTCOffsets": utils.GetUTCOffsets, + "NiceSeconds": utils.NiceSeconds, } render.AddFromFilesFuncs("login", helperFuncs, "templates/login.html") diff --git a/api/app-routes.go b/api/app-routes.go index a590898..d56a2f4 100644 --- a/api/app-routes.go +++ b/api/app-routes.go @@ -75,21 +75,24 @@ func (api *API) createAppResourcesRoute(routeName string, args ...map[string]any templateVarsBase["RouteName"] = routeName return func(c *gin.Context) { - rUser, _ := c.Get("AuthorizedUser") + var userID string + if rUser, _ := c.Get("AuthorizedUser"); rUser != nil { + userID = rUser.(string) + } // Copy Base & Update templateVars := gin.H{} for k, v := range templateVarsBase { templateVars[k] = v } - templateVars["User"] = rUser + templateVars["User"] = userID // Potential URL Parameters qParams := bindQueryParams(c) if routeName == "documents" { documents, err := api.DB.Queries.GetDocumentsWithStats(api.DB.Ctx, database.GetDocumentsWithStatsParams{ - UserID: rUser.(string), + UserID: userID, Offset: (*qParams.Page - 1) * *qParams.Limit, Limit: *qParams.Limit, }) @@ -99,6 +102,10 @@ func (api *API) createAppResourcesRoute(routeName string, args ...map[string]any return } + if err = api.getDocumentsWordCount(documents); err != nil { + log.Error("[createAppResourcesRoute] Unable to Get Word Counts: ", err) + } + templateVars["Data"] = documents } else if routeName == "document" { var rDocID requestDocumentID @@ -109,7 +116,7 @@ func (api *API) createAppResourcesRoute(routeName string, args ...map[string]any } document, err := api.DB.Queries.GetDocumentWithStats(api.DB.Ctx, database.GetDocumentWithStatsParams{ - UserID: rUser.(string), + UserID: userID, DocumentID: rDocID.DocumentID, }) if err != nil { @@ -118,11 +125,21 @@ func (api *API) createAppResourcesRoute(routeName string, args ...map[string]any return } + statistics := gin.H{ + "TotalTimeLeftSeconds": (document.TotalPages - document.CurrentPage) * document.SecondsPerPage, + "WordsPerMinute": "N/A", + } + + if document.Words != nil && *document.Words != 0 { + statistics["WordsPerMinute"] = (*document.Words / document.TotalPages * document.ReadPages) / (document.TotalTimeSeconds / 60.0) + } + templateVars["RelBase"] = "../" templateVars["Data"] = document + templateVars["Statistics"] = statistics } else if routeName == "activity" { activityFilter := database.GetActivityParams{ - UserID: rUser.(string), + UserID: userID, Offset: (*qParams.Page - 1) * *qParams.Limit, Limit: *qParams.Limit, } @@ -143,7 +160,7 @@ func (api *API) createAppResourcesRoute(routeName string, args ...map[string]any } else if routeName == "home" { start_time := time.Now() weekly_streak, err := api.DB.Queries.GetUserWindowStreaks(api.DB.Ctx, database.GetUserWindowStreaksParams{ - UserID: rUser.(string), + UserID: userID, Window: "WEEK", }) if err != nil { @@ -153,7 +170,7 @@ func (api *API) createAppResourcesRoute(routeName string, args ...map[string]any start_time = time.Now() daily_streak, err := api.DB.Queries.GetUserWindowStreaks(api.DB.Ctx, database.GetUserWindowStreaksParams{ - UserID: rUser.(string), + UserID: userID, Window: "DAY", }) if err != nil { @@ -162,11 +179,11 @@ func (api *API) createAppResourcesRoute(routeName string, args ...map[string]any log.Debug("GetUserWindowStreaks - DAY - ", time.Since(start_time)) start_time = time.Now() - database_info, _ := api.DB.Queries.GetDatabaseInfo(api.DB.Ctx, rUser.(string)) + database_info, _ := api.DB.Queries.GetDatabaseInfo(api.DB.Ctx, userID) log.Debug("GetDatabaseInfo - ", time.Since(start_time)) start_time = time.Now() - read_graph_data, _ := api.DB.Queries.GetDailyReadStats(api.DB.Ctx, rUser.(string)) + read_graph_data, _ := api.DB.Queries.GetDailyReadStats(api.DB.Ctx, userID) log.Debug("GetDailyReadStats - ", time.Since(start_time)) templateVars["Data"] = gin.H{ @@ -176,14 +193,14 @@ func (api *API) createAppResourcesRoute(routeName string, args ...map[string]any "GraphData": read_graph_data, } } else if routeName == "settings" { - user, err := api.DB.Queries.GetUser(api.DB.Ctx, rUser.(string)) + user, err := api.DB.Queries.GetUser(api.DB.Ctx, userID) if err != nil { log.Error("[createAppResourcesRoute] GetUser DB Error:", err) c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Request"}) return } - devices, err := api.DB.Queries.GetDevices(api.DB.Ctx, rUser.(string)) + devices, err := api.DB.Queries.GetDevices(api.DB.Ctx, userID) if err != nil { log.Error("[createAppResourcesRoute] GetDevices DB Error:", err) c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Request"}) @@ -248,16 +265,16 @@ func (api *API) getDocumentCover(c *gin.Context) { var coverFile string = "UNKNOWN" // Identify Documents & Save Covers - metadataResults, err := metadata.GetMetadata(metadata.MetadataInfo{ + metadataResults, err := metadata.SearchMetadata(metadata.GBOOK, metadata.MetadataInfo{ Title: document.Title, Author: document.Author, }) - if err == nil && len(metadataResults) > 0 && metadataResults[0].GBID != nil { + if err == nil && len(metadataResults) > 0 && metadataResults[0].ID != nil { firstResult := metadataResults[0] // Save Cover - fileName, err := metadata.SaveCover(*firstResult.GBID, coverDir, document.ID, false) + fileName, err := metadata.CacheCover(*firstResult.ID, coverDir, document.ID, false) if err == nil { coverFile = *fileName } @@ -268,8 +285,8 @@ func (api *API) getDocumentCover(c *gin.Context) { Title: firstResult.Title, Author: firstResult.Author, Description: firstResult.Description, - Gbid: firstResult.GBID, - Olid: firstResult.OLID, + Gbid: firstResult.ID, + Olid: nil, Isbn10: firstResult.ISBN10, Isbn13: firstResult.ISBN13, }); err != nil { @@ -368,7 +385,7 @@ func (api *API) editDocument(c *gin.Context) { coverFileName = &fileName } else if rDocEdit.CoverGBID != nil { var coverDir string = filepath.Join(api.Config.DataPath, "covers") - fileName, err := metadata.SaveCover(*rDocEdit.CoverGBID, coverDir, rDocID.DocumentID, true) + fileName, err := metadata.CacheCover(*rDocEdit.CoverGBID, coverDir, rDocID.DocumentID, true) if err == nil { coverFileName = fileName } @@ -456,7 +473,7 @@ func (api *API) identifyDocument(c *gin.Context) { } // Get Metadata - metadataResults, err := metadata.GetMetadata(metadata.MetadataInfo{ + metadataResults, err := metadata.SearchMetadata(metadata.GBOOK, metadata.MetadataInfo{ Title: rDocIdentify.Title, Author: rDocIdentify.Author, ISBN10: rDocIdentify.ISBN, @@ -471,8 +488,8 @@ func (api *API) identifyDocument(c *gin.Context) { Title: firstResult.Title, Author: firstResult.Author, Description: firstResult.Description, - Gbid: firstResult.GBID, - Olid: firstResult.OLID, + Gbid: firstResult.ID, + Olid: nil, Isbn10: firstResult.ISBN10, Isbn13: firstResult.ISBN13, }); err != nil { @@ -495,7 +512,17 @@ func (api *API) identifyDocument(c *gin.Context) { return } + statistics := gin.H{ + "TotalTimeLeftSeconds": (document.TotalPages - document.CurrentPage) * document.SecondsPerPage, + "WordsPerMinute": "N/A", + } + + if document.Words != nil && *document.Words != 0 { + statistics["WordsPerMinute"] = (*document.Words / document.TotalPages * document.ReadPages) / (document.TotalTimeSeconds / 60.0) + } + templateVars["Data"] = document + templateVars["Statistics"] = statistics c.HTML(http.StatusOK, "document", templateVars) } @@ -582,6 +609,45 @@ func (api *API) editSettings(c *gin.Context) { c.HTML(http.StatusOK, "settings", templateVars) } +func (api *API) getDocumentsWordCount(documents []database.GetDocumentsWithStatsRow) error { + // Do Transaction + tx, err := api.DB.DB.Begin() + if err != nil { + log.Error("[getDocumentsWordCount] Transaction Begin DB Error:", err) + return err + } + + // Defer & Start Transaction + defer tx.Rollback() + qtx := api.DB.Queries.WithTx(tx) + + for _, item := range documents { + if item.Words == nil && item.Filepath != nil { + filePath := filepath.Join(api.Config.DataPath, "documents", *item.Filepath) + wordCount, err := metadata.GetWordCount(filePath) + if err != nil { + log.Warn("[getDocumentsWordCount] Word Count Error - ", err) + } else { + if _, err := qtx.UpsertDocument(api.DB.Ctx, database.UpsertDocumentParams{ + ID: item.ID, + Words: &wordCount, + }); err != nil { + log.Error("[getDocumentsWordCount] UpsertDocument DB Error - ", err) + return err + } + } + } + } + + // Commit Transaction + if err := tx.Commit(); err != nil { + log.Error("[getDocumentsWordCount] Transaction Commit DB Error:", err) + return err + } + + return nil +} + func bindQueryParams(c *gin.Context) queryParams { var qParams queryParams c.BindQuery(&qParams) diff --git a/database/models.go b/database/models.go index 6692337..f199fb7 100644 --- a/database/models.go +++ b/database/models.go @@ -39,6 +39,7 @@ type Document struct { SeriesIndex *int64 `json:"series_index"` Lang *string `json:"lang"` Description *string `json:"description"` + Words *int64 `json:"words"` Gbid *string `json:"gbid"` Olid *string `json:"-"` Isbn10 *string `json:"isbn10"` diff --git a/database/query.sql b/database/query.sql index 2ee3389..91c8bd2 100644 --- a/database/query.sql +++ b/database/query.sql @@ -41,12 +41,13 @@ INSERT INTO documents ( series_index, lang, description, + words, olid, gbid, isbn10, isbn13 ) -VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT DO UPDATE SET md5 = COALESCE(excluded.md5, md5), @@ -58,6 +59,7 @@ SET series_index = COALESCE(excluded.series_index, series_index), lang = COALESCE(excluded.lang, lang), description = COALESCE(excluded.description, description), + words = COALESCE(excluded.words, words), olid = COALESCE(excluded.olid, olid), gbid = COALESCE(excluded.gbid, gbid), isbn10 = COALESCE(excluded.isbn10, isbn10), @@ -188,10 +190,15 @@ OFFSET $offset; WITH true_progress AS ( SELECT start_time AS last_read, - SUM(duration) / 60 AS total_time_minutes, + SUM(duration) AS total_time_seconds, document_id, current_page, total_pages, + + -- Determine Read Pages + COUNT(DISTINCT current_page) AS read_pages, + + -- Derive Percentage of Book ROUND(CAST(current_page AS REAL) / CAST(total_pages AS REAL) * 100, 2) AS percentage FROM activity WHERE user_id = $user_id @@ -205,13 +212,23 @@ SELECT CAST(IFNULL(current_page, 0) AS INTEGER) AS current_page, CAST(IFNULL(total_pages, 0) AS INTEGER) AS total_pages, - CAST(IFNULL(total_time_minutes, 0) AS INTEGER) AS total_time_minutes, + CAST(IFNULL(total_time_seconds, 0) AS INTEGER) AS total_time_seconds, CAST(DATETIME(IFNULL(last_read, "1970-01-01"), time_offset) AS TEXT) AS last_read, + CAST(IFNULL(read_pages, 0) AS INTEGER) AS read_pages, + -- Calculate Seconds / Page + -- 1. Calculate Total Time in Seconds (Sum Duration in Activity) + -- 2. Divide by Read Pages (Distinct Pages in Activity) CAST(CASE - WHEN percentage > 97.0 THEN 100.0 - WHEN percentage IS NULL THEN 0.0 - ELSE percentage + WHEN total_time_seconds IS NULL THEN 0.0 + ELSE ROUND(CAST(total_time_seconds AS REAL) / CAST(read_pages AS REAL)) + END AS INTEGER) AS seconds_per_page, + + -- Arbitrarily >97% is Complete + CAST(CASE + WHEN percentage > 97.0 THEN 100.0 + WHEN percentage IS NULL THEN 0.0 + ELSE percentage END AS REAL) AS percentage FROM documents @@ -225,7 +242,7 @@ LIMIT 1; WITH true_progress AS ( SELECT start_time AS last_read, - SUM(duration) / 60 AS total_time_minutes, + SUM(duration) AS total_time_seconds, document_id, current_page, total_pages, @@ -240,7 +257,7 @@ SELECT CAST(IFNULL(current_page, 0) AS INTEGER) AS current_page, CAST(IFNULL(total_pages, 0) AS INTEGER) AS total_pages, - CAST(IFNULL(total_time_minutes, 0) AS INTEGER) AS total_time_minutes, + CAST(IFNULL(total_time_seconds, 0) AS INTEGER) AS total_time_seconds, CAST(DATETIME(IFNULL(last_read, "1970-01-01"), time_offset) AS TEXT) AS last_read, CAST(CASE diff --git a/database/query.sql.go b/database/query.sql.go index 64ba745..7dc22e7 100644 --- a/database/query.sql.go +++ b/database/query.sql.go @@ -417,7 +417,7 @@ func (q *Queries) GetDevices(ctx context.Context, userID string) ([]GetDevicesRo } const getDocument = `-- name: GetDocument :one -SELECT id, md5, filepath, coverfile, title, author, series, series_index, lang, description, gbid, olid, isbn10, isbn13, synced, deleted, updated_at, created_at FROM documents +SELECT id, md5, filepath, coverfile, title, author, series, series_index, lang, description, words, gbid, olid, isbn10, isbn13, synced, deleted, updated_at, created_at FROM documents WHERE id = ?1 LIMIT 1 ` @@ -435,6 +435,7 @@ func (q *Queries) GetDocument(ctx context.Context, documentID string) (Document, &i.SeriesIndex, &i.Lang, &i.Description, + &i.Words, &i.Gbid, &i.Olid, &i.Isbn10, @@ -543,10 +544,15 @@ const getDocumentWithStats = `-- name: GetDocumentWithStats :one WITH true_progress AS ( SELECT start_time AS last_read, - SUM(duration) / 60 AS total_time_minutes, + SUM(duration) AS total_time_seconds, document_id, current_page, total_pages, + + -- Determine Read Pages + COUNT(DISTINCT current_page) AS read_pages, + + -- Derive Percentage of Book ROUND(CAST(current_page AS REAL) / CAST(total_pages AS REAL) * 100, 2) AS percentage FROM activity WHERE user_id = ?1 @@ -556,17 +562,27 @@ WITH true_progress AS ( LIMIT 1 ) SELECT - documents.id, documents.md5, documents.filepath, documents.coverfile, documents.title, documents.author, documents.series, documents.series_index, documents.lang, documents.description, documents.gbid, documents.olid, documents.isbn10, documents.isbn13, documents.synced, documents.deleted, documents.updated_at, documents.created_at, + documents.id, documents.md5, documents.filepath, documents.coverfile, documents.title, documents.author, documents.series, documents.series_index, documents.lang, documents.description, documents.words, documents.gbid, documents.olid, documents.isbn10, documents.isbn13, documents.synced, documents.deleted, documents.updated_at, documents.created_at, CAST(IFNULL(current_page, 0) AS INTEGER) AS current_page, CAST(IFNULL(total_pages, 0) AS INTEGER) AS total_pages, - CAST(IFNULL(total_time_minutes, 0) AS INTEGER) AS total_time_minutes, + CAST(IFNULL(total_time_seconds, 0) AS INTEGER) AS total_time_seconds, CAST(DATETIME(IFNULL(last_read, "1970-01-01"), time_offset) AS TEXT) AS last_read, + CAST(IFNULL(read_pages, 0) AS INTEGER) AS read_pages, + -- Calculate Seconds / Page + -- 1. Calculate Total Time in Seconds (Sum Duration in Activity) + -- 2. Divide by Read Pages (Distinct Pages in Activity) CAST(CASE - WHEN percentage > 97.0 THEN 100.0 - WHEN percentage IS NULL THEN 0.0 - ELSE percentage + WHEN total_time_seconds IS NULL THEN 0.0 + ELSE ROUND(CAST(total_time_seconds AS REAL) / CAST(read_pages AS REAL)) + END AS INTEGER) AS seconds_per_page, + + -- Arbitrarily >97% is Complete + CAST(CASE + WHEN percentage > 97.0 THEN 100.0 + WHEN percentage IS NULL THEN 0.0 + ELSE percentage END AS REAL) AS percentage FROM documents @@ -593,6 +609,7 @@ type GetDocumentWithStatsRow struct { SeriesIndex *int64 `json:"series_index"` Lang *string `json:"lang"` Description *string `json:"description"` + Words *int64 `json:"words"` Gbid *string `json:"gbid"` Olid *string `json:"-"` Isbn10 *string `json:"isbn10"` @@ -603,8 +620,10 @@ type GetDocumentWithStatsRow struct { CreatedAt time.Time `json:"created_at"` CurrentPage int64 `json:"current_page"` TotalPages int64 `json:"total_pages"` - TotalTimeMinutes int64 `json:"total_time_minutes"` + TotalTimeSeconds int64 `json:"total_time_seconds"` LastRead string `json:"last_read"` + ReadPages int64 `json:"read_pages"` + SecondsPerPage int64 `json:"seconds_per_page"` Percentage float64 `json:"percentage"` } @@ -622,6 +641,7 @@ func (q *Queries) GetDocumentWithStats(ctx context.Context, arg GetDocumentWithS &i.SeriesIndex, &i.Lang, &i.Description, + &i.Words, &i.Gbid, &i.Olid, &i.Isbn10, @@ -632,15 +652,17 @@ func (q *Queries) GetDocumentWithStats(ctx context.Context, arg GetDocumentWithS &i.CreatedAt, &i.CurrentPage, &i.TotalPages, - &i.TotalTimeMinutes, + &i.TotalTimeSeconds, &i.LastRead, + &i.ReadPages, + &i.SecondsPerPage, &i.Percentage, ) return i, err } const getDocuments = `-- name: GetDocuments :many -SELECT id, md5, filepath, coverfile, title, author, series, series_index, lang, description, gbid, olid, isbn10, isbn13, synced, deleted, updated_at, created_at FROM documents +SELECT id, md5, filepath, coverfile, title, author, series, series_index, lang, description, words, gbid, olid, isbn10, isbn13, synced, deleted, updated_at, created_at FROM documents ORDER BY created_at DESC LIMIT ?2 OFFSET ?1 @@ -671,6 +693,7 @@ func (q *Queries) GetDocuments(ctx context.Context, arg GetDocumentsParams) ([]D &i.SeriesIndex, &i.Lang, &i.Description, + &i.Words, &i.Gbid, &i.Olid, &i.Isbn10, @@ -697,7 +720,7 @@ const getDocumentsWithStats = `-- name: GetDocumentsWithStats :many WITH true_progress AS ( SELECT start_time AS last_read, - SUM(duration) / 60 AS total_time_minutes, + SUM(duration) AS total_time_seconds, document_id, current_page, total_pages, @@ -708,11 +731,11 @@ WITH true_progress AS ( HAVING MAX(start_time) ) SELECT - documents.id, documents.md5, documents.filepath, documents.coverfile, documents.title, documents.author, documents.series, documents.series_index, documents.lang, documents.description, documents.gbid, documents.olid, documents.isbn10, documents.isbn13, documents.synced, documents.deleted, documents.updated_at, documents.created_at, + documents.id, documents.md5, documents.filepath, documents.coverfile, documents.title, documents.author, documents.series, documents.series_index, documents.lang, documents.description, documents.words, documents.gbid, documents.olid, documents.isbn10, documents.isbn13, documents.synced, documents.deleted, documents.updated_at, documents.created_at, CAST(IFNULL(current_page, 0) AS INTEGER) AS current_page, CAST(IFNULL(total_pages, 0) AS INTEGER) AS total_pages, - CAST(IFNULL(total_time_minutes, 0) AS INTEGER) AS total_time_minutes, + CAST(IFNULL(total_time_seconds, 0) AS INTEGER) AS total_time_seconds, CAST(DATETIME(IFNULL(last_read, "1970-01-01"), time_offset) AS TEXT) AS last_read, CAST(CASE @@ -747,6 +770,7 @@ type GetDocumentsWithStatsRow struct { SeriesIndex *int64 `json:"series_index"` Lang *string `json:"lang"` Description *string `json:"description"` + Words *int64 `json:"words"` Gbid *string `json:"gbid"` Olid *string `json:"-"` Isbn10 *string `json:"isbn10"` @@ -757,7 +781,7 @@ type GetDocumentsWithStatsRow struct { CreatedAt time.Time `json:"created_at"` CurrentPage int64 `json:"current_page"` TotalPages int64 `json:"total_pages"` - TotalTimeMinutes int64 `json:"total_time_minutes"` + TotalTimeSeconds int64 `json:"total_time_seconds"` LastRead string `json:"last_read"` Percentage float64 `json:"percentage"` } @@ -782,6 +806,7 @@ func (q *Queries) GetDocumentsWithStats(ctx context.Context, arg GetDocumentsWit &i.SeriesIndex, &i.Lang, &i.Description, + &i.Words, &i.Gbid, &i.Olid, &i.Isbn10, @@ -792,7 +817,7 @@ func (q *Queries) GetDocumentsWithStats(ctx context.Context, arg GetDocumentsWit &i.CreatedAt, &i.CurrentPage, &i.TotalPages, - &i.TotalTimeMinutes, + &i.TotalTimeSeconds, &i.LastRead, &i.Percentage, ); err != nil { @@ -830,7 +855,7 @@ func (q *Queries) GetLastActivity(ctx context.Context, arg GetLastActivityParams } const getMissingDocuments = `-- name: GetMissingDocuments :many -SELECT documents.id, documents.md5, documents.filepath, documents.coverfile, documents.title, documents.author, documents.series, documents.series_index, documents.lang, documents.description, documents.gbid, documents.olid, documents.isbn10, documents.isbn13, documents.synced, documents.deleted, documents.updated_at, documents.created_at FROM documents +SELECT documents.id, documents.md5, documents.filepath, documents.coverfile, documents.title, documents.author, documents.series, documents.series_index, documents.lang, documents.description, documents.words, documents.gbid, documents.olid, documents.isbn10, documents.isbn13, documents.synced, documents.deleted, documents.updated_at, documents.created_at FROM documents WHERE documents.filepath IS NOT NULL AND documents.deleted = false @@ -867,6 +892,7 @@ func (q *Queries) GetMissingDocuments(ctx context.Context, documentIds []string) &i.SeriesIndex, &i.Lang, &i.Description, + &i.Words, &i.Gbid, &i.Olid, &i.Isbn10, @@ -1157,7 +1183,7 @@ UPDATE documents SET deleted = ?1 WHERE id = ?2 -RETURNING id, md5, filepath, coverfile, title, author, series, series_index, lang, description, gbid, olid, isbn10, isbn13, synced, deleted, updated_at, created_at +RETURNING id, md5, filepath, coverfile, title, author, series, series_index, lang, description, words, gbid, olid, isbn10, isbn13, synced, deleted, updated_at, created_at ` type UpdateDocumentDeletedParams struct { @@ -1179,6 +1205,7 @@ func (q *Queries) UpdateDocumentDeleted(ctx context.Context, arg UpdateDocumentD &i.SeriesIndex, &i.Lang, &i.Description, + &i.Words, &i.Gbid, &i.Olid, &i.Isbn10, @@ -1196,7 +1223,7 @@ UPDATE documents SET synced = ?1 WHERE id = ?2 -RETURNING id, md5, filepath, coverfile, title, author, series, series_index, lang, description, gbid, olid, isbn10, isbn13, synced, deleted, updated_at, created_at +RETURNING id, md5, filepath, coverfile, title, author, series, series_index, lang, description, words, gbid, olid, isbn10, isbn13, synced, deleted, updated_at, created_at ` type UpdateDocumentSyncParams struct { @@ -1218,6 +1245,7 @@ func (q *Queries) UpdateDocumentSync(ctx context.Context, arg UpdateDocumentSync &i.SeriesIndex, &i.Lang, &i.Description, + &i.Words, &i.Gbid, &i.Olid, &i.Isbn10, @@ -1338,12 +1366,13 @@ INSERT INTO documents ( series_index, lang, description, + words, olid, gbid, isbn10, isbn13 ) -VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) +VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT DO UPDATE SET md5 = COALESCE(excluded.md5, md5), @@ -1355,11 +1384,12 @@ SET series_index = COALESCE(excluded.series_index, series_index), lang = COALESCE(excluded.lang, lang), description = COALESCE(excluded.description, description), + words = COALESCE(excluded.words, words), olid = COALESCE(excluded.olid, olid), gbid = COALESCE(excluded.gbid, gbid), isbn10 = COALESCE(excluded.isbn10, isbn10), isbn13 = COALESCE(excluded.isbn13, isbn13) -RETURNING id, md5, filepath, coverfile, title, author, series, series_index, lang, description, gbid, olid, isbn10, isbn13, synced, deleted, updated_at, created_at +RETURNING id, md5, filepath, coverfile, title, author, series, series_index, lang, description, words, gbid, olid, isbn10, isbn13, synced, deleted, updated_at, created_at ` type UpsertDocumentParams struct { @@ -1373,6 +1403,7 @@ type UpsertDocumentParams struct { SeriesIndex *int64 `json:"series_index"` Lang *string `json:"lang"` Description *string `json:"description"` + Words *int64 `json:"words"` Olid *string `json:"-"` Gbid *string `json:"gbid"` Isbn10 *string `json:"isbn10"` @@ -1391,6 +1422,7 @@ func (q *Queries) UpsertDocument(ctx context.Context, arg UpsertDocumentParams) arg.SeriesIndex, arg.Lang, arg.Description, + arg.Words, arg.Olid, arg.Gbid, arg.Isbn10, @@ -1408,6 +1440,7 @@ func (q *Queries) UpsertDocument(ctx context.Context, arg UpsertDocumentParams) &i.SeriesIndex, &i.Lang, &i.Description, + &i.Words, &i.Gbid, &i.Olid, &i.Isbn10, diff --git a/database/schema.sql b/database/schema.sql index db62b06..eb04bc5 100644 --- a/database/schema.sql +++ b/database/schema.sql @@ -25,6 +25,7 @@ CREATE TABLE IF NOT EXISTS documents ( series_index INTEGER, lang TEXT, description TEXT, + words INTEGER, gbid TEXT, olid TEXT, diff --git a/metadata/epub.go b/metadata/epub.go new file mode 100644 index 0000000..7d8261b --- /dev/null +++ b/metadata/epub.go @@ -0,0 +1,330 @@ +/* +Package epub provides basic support for reading EPUB archives. +Adapted from: https://github.com/taylorskalyo/goreader +*/ +package metadata + +import ( + "archive/zip" + "bytes" + "encoding/xml" + "errors" + "io" + "os" + "path" + "strings" + + "golang.org/x/net/html" +) + +const containerPath = "META-INF/container.xml" + +var ( + // ErrNoRootfile occurs when there are no rootfile entries found in + // container.xml. + ErrNoRootfile = errors.New("epub: no rootfile found in container") + + // ErrBadRootfile occurs when container.xml references a rootfile that does + // not exist in the zip. + ErrBadRootfile = errors.New("epub: container references non-existent rootfile") + + // ErrNoItemref occurrs when a content.opf contains a spine without any + // itemref entries. + ErrNoItemref = errors.New("epub: no itemrefs found in spine") + + // ErrBadItemref occurs when an itemref entry in content.opf references an + // item that does not exist in the manifest. + ErrBadItemref = errors.New("epub: itemref references non-existent item") + + // ErrBadManifest occurs when a manifest in content.opf references an item + // that does not exist in the zip. + ErrBadManifest = errors.New("epub: manifest references non-existent item") +) + +// Reader represents a readable epub file. +type Reader struct { + Container + files map[string]*zip.File +} + +// ReadCloser represents a readable epub file that can be closed. +type ReadCloser struct { + Reader + f *os.File +} + +// Rootfile contains the location of a content.opf package file. +type Rootfile struct { + FullPath string `xml:"full-path,attr"` + Package +} + +// Container serves as a directory of Rootfiles. +type Container struct { + Rootfiles []*Rootfile `xml:"rootfiles>rootfile"` +} + +// Package represents an epub content.opf file. +type Package struct { + Metadata + Manifest + Spine +} + +// Metadata contains publishing information about the epub. +type Metadata struct { + Title string `xml:"metadata>title"` + Language string `xml:"metadata>language"` + Identifier string `xml:"metadata>idenifier"` + Creator string `xml:"metadata>creator"` + Contributor string `xml:"metadata>contributor"` + Publisher string `xml:"metadata>publisher"` + Subject string `xml:"metadata>subject"` + Description string `xml:"metadata>description"` + Event []struct { + Name string `xml:"event,attr"` + Date string `xml:",innerxml"` + } `xml:"metadata>date"` + Type string `xml:"metadata>type"` + Format string `xml:"metadata>format"` + Source string `xml:"metadata>source"` + Relation string `xml:"metadata>relation"` + Coverage string `xml:"metadata>coverage"` + Rights string `xml:"metadata>rights"` +} + +// Manifest lists every file that is part of the epub. +type Manifest struct { + Items []Item `xml:"manifest>item"` +} + +// Item represents a file stored in the epub. +type Item struct { + ID string `xml:"id,attr"` + HREF string `xml:"href,attr"` + MediaType string `xml:"media-type,attr"` + f *zip.File +} + +// Spine defines the reading order of the epub documents. +type Spine struct { + Itemrefs []Itemref `xml:"spine>itemref"` +} + +// Itemref points to an Item. +type Itemref struct { + IDREF string `xml:"idref,attr"` + *Item +} + +// OpenEPUBReader will open the epub file specified by name and return a +// ReadCloser. +func OpenEPUBReader(name string) (*ReadCloser, error) { + f, err := os.Open(name) + if err != nil { + return nil, err + } + + rc := new(ReadCloser) + rc.f = f + + fi, err := f.Stat() + if err != nil { + f.Close() + return nil, err + } + + z, err := zip.NewReader(f, fi.Size()) + if err != nil { + return nil, err + } + + if err = rc.init(z); err != nil { + return nil, err + } + + return rc, nil +} + +// NewReader returns a new Reader reading from ra, which is assumed to have the +// given size in bytes. +func NewReader(ra io.ReaderAt, size int64) (*Reader, error) { + z, err := zip.NewReader(ra, size) + if err != nil { + return nil, err + } + + r := new(Reader) + if err = r.init(z); err != nil { + return nil, err + } + + return r, nil +} + +func (r *Reader) init(z *zip.Reader) error { + // Create a file lookup table + r.files = make(map[string]*zip.File) + for _, f := range z.File { + r.files[f.Name] = f + } + + err := r.setContainer() + if err != nil { + return err + } + err = r.setPackages() + if err != nil { + return err + } + err = r.setItems() + if err != nil { + return err + } + + return nil +} + +// setContainer unmarshals the epub's container.xml file. +func (r *Reader) setContainer() error { + f, err := r.files[containerPath].Open() + if err != nil { + return err + } + + var b bytes.Buffer + _, err = io.Copy(&b, f) + if err != nil { + return err + } + + err = xml.Unmarshal(b.Bytes(), &r.Container) + if err != nil { + return err + } + + if len(r.Container.Rootfiles) < 1 { + return ErrNoRootfile + } + + return nil +} + +// setPackages unmarshal's each of the epub's content.opf files. +func (r *Reader) setPackages() error { + for _, rf := range r.Container.Rootfiles { + if r.files[rf.FullPath] == nil { + return ErrBadRootfile + } + + f, err := r.files[rf.FullPath].Open() + if err != nil { + return err + } + + var b bytes.Buffer + _, err = io.Copy(&b, f) + if err != nil { + return err + } + + err = xml.Unmarshal(b.Bytes(), &rf.Package) + if err != nil { + return err + } + } + + return nil +} + +// setItems associates Itemrefs with their respective Item and Items with +// their zip.File. +func (r *Reader) setItems() error { + itemrefCount := 0 + for _, rf := range r.Container.Rootfiles { + itemMap := make(map[string]*Item) + for i := range rf.Manifest.Items { + item := &rf.Manifest.Items[i] + itemMap[item.ID] = item + + abs := path.Join(path.Dir(rf.FullPath), item.HREF) + item.f = r.files[abs] + } + + for i := range rf.Spine.Itemrefs { + itemref := &rf.Spine.Itemrefs[i] + itemref.Item = itemMap[itemref.IDREF] + if itemref.Item == nil { + return ErrBadItemref + } + } + itemrefCount += len(rf.Spine.Itemrefs) + } + + if itemrefCount < 1 { + return ErrNoItemref + } + + return nil +} + +// Open returns a ReadCloser that provides access to the Items's contents. +// Multiple items may be read concurrently. +func (item *Item) Open() (r io.ReadCloser, err error) { + if item.f == nil { + return nil, ErrBadManifest + } + + return item.f.Open() +} + +// Close closes the epub file, rendering it unusable for I/O. +func (rc *ReadCloser) Close() { + rc.f.Close() +} + +// Hehe +func (rf *Rootfile) CountWords() int64 { + var completeCount int64 + for _, item := range rf.Spine.Itemrefs { + f, _ := item.Open() + tokenizer := html.NewTokenizer(f) + completeCount = completeCount + countWords(*tokenizer) + } + + return completeCount +} + +func countWords(tokenizer html.Tokenizer) int64 { + var err error + var totalWords int64 + for { + tokenType := tokenizer.Next() + token := tokenizer.Token() + if tokenType == html.TextToken { + currStr := string(token.Data) + totalWords = totalWords + int64(len(strings.Fields(currStr))) + } else if tokenType == html.ErrorToken { + err = tokenizer.Err() + } + if err == io.EOF { + return totalWords + } else if err != nil { + return 0 + } + } +} + +/* +func main() { + rc, err := OpenEPUBReader("test.epub") + if err != nil { + log.Fatal(err) + } + + rf := rc.Rootfiles[0] + + totalWords := rf.CountWords() + log.Info("WOAH WORDS:", totalWords) +} +*/ diff --git a/metadata/gbooks.go b/metadata/gbooks.go new file mode 100644 index 0000000..18db85a --- /dev/null +++ b/metadata/gbooks.go @@ -0,0 +1,200 @@ +package metadata + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strings" + + log "github.com/sirupsen/logrus" +) + +type gBooksIdentifiers struct { + Type string `json:"type"` + Identifier string `json:"identifier"` +} + +type gBooksInfo struct { + Title string `json:"title"` + Authors []string `json:"authors"` + Description string `json:"description"` + Identifiers []gBooksIdentifiers `json:"industryIdentifiers"` +} + +type gBooksQueryItem struct { + ID string `json:"id"` + Info gBooksInfo `json:"volumeInfo"` +} + +type gBooksQueryResponse struct { + TotalItems int `json:"totalItems"` + Items []gBooksQueryItem `json:"items"` +} + +const GBOOKS_QUERY_URL string = "https://www.googleapis.com/books/v1/volumes?q=%s" +const GBOOKS_GBID_INFO_URL string = "https://www.googleapis.com/books/v1/volumes/%s" +const GBOOKS_GBID_COVER_URL string = "https://books.google.com/books/content/images/frontcover/%s?fife=w480-h690" + +func getGBooksMetadata(metadataSearch MetadataInfo) ([]MetadataInfo, error) { + var queryResults []gBooksQueryItem + if metadataSearch.ID != nil { + // Use GBID + resp, err := performGBIDRequest(*metadataSearch.ID) + if err != nil { + return nil, err + } + + queryResults = []gBooksQueryItem{*resp} + } else if metadataSearch.ISBN13 != nil { + searchQuery := "isbn:" + *metadataSearch.ISBN13 + resp, err := performSearchRequest(searchQuery) + if err != nil { + return nil, err + } + + queryResults = resp.Items + } else if metadataSearch.ISBN10 != nil { + searchQuery := "isbn:" + *metadataSearch.ISBN10 + resp, err := performSearchRequest(searchQuery) + if err != nil { + return nil, err + } + + queryResults = resp.Items + } else if metadataSearch.Title != nil || metadataSearch.Author != nil { + var searchQuery string + if metadataSearch.Title != nil { + searchQuery = searchQuery + *metadataSearch.Title + } + if metadataSearch.Author != nil { + searchQuery = searchQuery + " " + *metadataSearch.Author + } + + // Escape & Trim + searchQuery = url.QueryEscape(strings.TrimSpace(searchQuery)) + resp, err := performSearchRequest(searchQuery) + if err != nil { + return nil, err + } + + queryResults = resp.Items + } else { + return nil, errors.New("Invalid Data") + } + + // Normalize Data + allMetadata := []MetadataInfo{} + for i := range queryResults { + item := queryResults[i] // Range Value Pointer Issue + itemResult := MetadataInfo{ + ID: &item.ID, + Title: &item.Info.Title, + Description: &item.Info.Description, + } + + if len(item.Info.Authors) > 0 { + itemResult.Author = &item.Info.Authors[0] + } + + for i := range item.Info.Identifiers { + item := item.Info.Identifiers[i] // Range Value Pointer Issue + + if itemResult.ISBN10 != nil && itemResult.ISBN13 != nil { + break + } else if itemResult.ISBN10 == nil && item.Type == "ISBN_10" { + itemResult.ISBN10 = &item.Identifier + } else if itemResult.ISBN13 == nil && item.Type == "ISBN_13" { + itemResult.ISBN13 = &item.Identifier + } + } + + allMetadata = append(allMetadata, itemResult) + } + + return allMetadata, nil +} + +func saveGBooksCover(gbid string, coverFilePath string, overwrite bool) error { + // Validate File Doesn't Exists + _, err := os.Stat(coverFilePath) + if err == nil && overwrite == false { + log.Warn("[saveGBooksCover] File Alreads Exists") + return nil + } + + // Create File + out, err := os.Create(coverFilePath) + if err != nil { + log.Error("[saveGBooksCover] File Create Error") + return errors.New("File Failure") + } + defer out.Close() + + // Download File + log.Info("[saveGBooksCover] Downloading Cover") + coverURL := fmt.Sprintf(GBOOKS_GBID_COVER_URL, gbid) + resp, err := http.Get(coverURL) + if err != nil { + log.Error("[saveGBooksCover] Cover URL API Failure") + return errors.New("API Failure") + } + defer resp.Body.Close() + + // Copy File to Disk + log.Info("[saveGBooksCover] Saving Cover") + _, err = io.Copy(out, resp.Body) + if err != nil { + log.Error("[saveGBooksCover] File Copy Error") + return errors.New("File Failure") + } + + return nil +} + +func performSearchRequest(searchQuery string) (*gBooksQueryResponse, error) { + apiQuery := fmt.Sprintf(GBOOKS_QUERY_URL, searchQuery) + log.Info("[performSearchRequest] Acquiring Metadata: ", apiQuery) + resp, err := http.Get(apiQuery) + if err != nil { + log.Error("[performSearchRequest] Google Books Query URL API Failure") + return nil, errors.New("API Failure") + } + + parsedResp := gBooksQueryResponse{} + err = json.NewDecoder(resp.Body).Decode(&parsedResp) + if err != nil { + log.Error("[performSearchRequest] Google Books Query API Decode Failure") + return nil, errors.New("API Failure") + } + + if len(parsedResp.Items) == 0 { + log.Warn("[performSearchRequest] No Results") + return nil, errors.New("No Results") + } + + return &parsedResp, nil +} + +func performGBIDRequest(id string) (*gBooksQueryItem, error) { + apiQuery := fmt.Sprintf(GBOOKS_GBID_INFO_URL, id) + + log.Info("[performGBIDRequest] Acquiring CoverID") + resp, err := http.Get(apiQuery) + if err != nil { + log.Error("[performGBIDRequest] Cover URL API Failure") + return nil, errors.New("API Failure") + } + + parsedResp := gBooksQueryItem{} + err = json.NewDecoder(resp.Body).Decode(&parsedResp) + if err != nil { + log.Error("[performGBIDRequest] Google Books ID API Decode Failure") + return nil, errors.New("API Failure") + } + + return &parsedResp, nil +} diff --git a/metadata/metadata.go b/metadata/metadata.go index 6e15aad..c3df694 100644 --- a/metadata/metadata.go +++ b/metadata/metadata.go @@ -1,217 +1,72 @@ package metadata import ( - "encoding/json" "errors" "fmt" - "io" - "net/http" - "net/url" - "os" "path/filepath" - "strings" - log "github.com/sirupsen/logrus" + "github.com/gabriel-vasile/mimetype" +) + +type Source int + +const ( + GBOOK Source = iota + OLIB ) type MetadataInfo struct { + ID *string Title *string Author *string Description *string - GBID *string - OLID *string ISBN10 *string ISBN13 *string } -type gBooksIdentifiers struct { - Type string `json:"type"` - Identifier string `json:"identifier"` -} - -type gBooksInfo struct { - Title string `json:"title"` - Authors []string `json:"authors"` - Description string `json:"description"` - Identifiers []gBooksIdentifiers `json:"industryIdentifiers"` -} - -type gBooksQueryItem struct { - ID string `json:"id"` - Info gBooksInfo `json:"volumeInfo"` -} - -type gBooksQueryResponse struct { - TotalItems int `json:"totalItems"` - Items []gBooksQueryItem `json:"items"` -} - -const GBOOKS_QUERY_URL string = "https://www.googleapis.com/books/v1/volumes?q=%s" -const GBOOKS_GBID_INFO_URL string = "https://www.googleapis.com/books/v1/volumes/%s" -const GBOOKS_GBID_COVER_URL string = "https://books.google.com/books/content/images/frontcover/%s?fife=w480-h690" - -func GetMetadata(metadataSearch MetadataInfo) ([]MetadataInfo, error) { - var queryResults []gBooksQueryItem - if metadataSearch.GBID != nil { - // Use GBID - resp, err := performGBIDRequest(*metadataSearch.GBID) - if err != nil { - return nil, err - } - - queryResults = []gBooksQueryItem{*resp} - } else if metadataSearch.ISBN13 != nil { - searchQuery := "isbn:" + *metadataSearch.ISBN13 - resp, err := performSearchRequest(searchQuery) - if err != nil { - return nil, err - } - - queryResults = resp.Items - } else if metadataSearch.ISBN10 != nil { - searchQuery := "isbn:" + *metadataSearch.ISBN10 - resp, err := performSearchRequest(searchQuery) - if err != nil { - return nil, err - } - - queryResults = resp.Items - } else if metadataSearch.Title != nil || metadataSearch.Author != nil { - var searchQuery string - if metadataSearch.Title != nil { - searchQuery = searchQuery + *metadataSearch.Title - } - if metadataSearch.Author != nil { - searchQuery = searchQuery + " " + *metadataSearch.Author - } - - // Escape & Trim - searchQuery = url.QueryEscape(strings.TrimSpace(searchQuery)) - resp, err := performSearchRequest(searchQuery) - if err != nil { - return nil, err - } - - queryResults = resp.Items - } else { - return nil, errors.New("Invalid Data") - } - - // Normalize Data - allMetadata := []MetadataInfo{} - for i := range queryResults { - item := queryResults[i] // Range Value Pointer Issue - itemResult := MetadataInfo{ - GBID: &item.ID, - Title: &item.Info.Title, - Description: &item.Info.Description, - } - - if len(item.Info.Authors) > 0 { - itemResult.Author = &item.Info.Authors[0] - } - - for i := range item.Info.Identifiers { - item := item.Info.Identifiers[i] // Range Value Pointer Issue - - if itemResult.ISBN10 != nil && itemResult.ISBN13 != nil { - break - } else if itemResult.ISBN10 == nil && item.Type == "ISBN_10" { - itemResult.ISBN10 = &item.Identifier - } else if itemResult.ISBN13 == nil && item.Type == "ISBN_13" { - itemResult.ISBN13 = &item.Identifier - } - } - - allMetadata = append(allMetadata, itemResult) - } - - return allMetadata, nil -} - -func SaveCover(gbid string, coverDir string, documentID string, overwrite bool) (*string, error) { - - // Google Books -> JPG +func CacheCover(gbid string, coverDir string, documentID string, overwrite bool) (*string, error) { + // Get Filepath coverFile := "." + filepath.Clean(fmt.Sprintf("/%s.jpg", documentID)) coverFilePath := filepath.Join(coverDir, coverFile) - // Validate File Doesn't Exists - _, err := os.Stat(coverFilePath) - if err == nil && overwrite == false { - log.Warn("[SaveCover] File Alreads Exists") - return &coverFile, nil + // Save Google Books + if err := saveGBooksCover(gbid, coverFilePath, overwrite); err != nil { + return nil, err } - // Create File - out, err := os.Create(coverFilePath) - if err != nil { - log.Error("[SaveCover] File Create Error") - return nil, errors.New("File Failure") - } - defer out.Close() + // TODO - Refactor & Allow Open Library / Alternative Sources - // Download File - log.Info("[SaveCover] Downloading Cover") - coverURL := fmt.Sprintf(GBOOKS_GBID_COVER_URL, gbid) - resp, err := http.Get(coverURL) - if err != nil { - log.Error("[SaveCover] Cover URL API Failure") - return nil, errors.New("API Failure") - } - defer resp.Body.Close() - - // Copy File to Disk - log.Info("[SaveCover] Saving Cover") - _, err = io.Copy(out, resp.Body) - if err != nil { - log.Error("[SaveCover] File Copy Error") - return nil, errors.New("File Failure") - } - - // Return FilePath return &coverFile, nil } -func performSearchRequest(searchQuery string) (*gBooksQueryResponse, error) { - apiQuery := fmt.Sprintf(GBOOKS_QUERY_URL, searchQuery) - log.Info("[performSearchRequest] Acquiring Metadata: ", apiQuery) - resp, err := http.Get(apiQuery) - if err != nil { - log.Error("[performSearchRequest] Google Books Query URL API Failure") - return nil, errors.New("API Failure") - } +func SearchMetadata(s Source, metadataSearch MetadataInfo) ([]MetadataInfo, error) { + switch s { + case GBOOK: + return getGBooksMetadata(metadataSearch) + case OLIB: + return nil, errors.New("Not implemented") + default: + return nil, errors.New("Not implemented") - parsedResp := gBooksQueryResponse{} - err = json.NewDecoder(resp.Body).Decode(&parsedResp) - if err != nil { - log.Error("[performSearchRequest] Google Books Query API Decode Failure") - return nil, errors.New("API Failure") } - - if len(parsedResp.Items) == 0 { - log.Warn("[performSearchRequest] No Results") - return nil, errors.New("No Results") - } - - return &parsedResp, nil } -func performGBIDRequest(id string) (*gBooksQueryItem, error) { - apiQuery := fmt.Sprintf(GBOOKS_GBID_INFO_URL, id) - - log.Info("[performGBIDRequest] Acquiring CoverID") - resp, err := http.Get(apiQuery) +func GetWordCount(filepath string) (int64, error) { + fileMime, err := mimetype.DetectFile(filepath) if err != nil { - log.Error("[performGBIDRequest] Cover URL API Failure") - return nil, errors.New("API Failure") + return 0, err } - parsedResp := gBooksQueryItem{} - err = json.NewDecoder(resp.Body).Decode(&parsedResp) - if err != nil { - log.Error("[performGBIDRequest] Google Books ID API Decode Failure") - return nil, errors.New("API Failure") - } + if fileExtension := fileMime.Extension(); fileExtension == ".epub" { + rc, err := OpenEPUBReader(filepath) + if err != nil { + return 0, err + } - return &parsedResp, nil + rf := rc.Rootfiles[0] + totalWords := rf.CountWords() + return totalWords, nil + } else { + return 0, errors.New("Invalid Extension") + } } diff --git a/sqlc.yaml b/sqlc.yaml index e94f307..79db8ba 100644 --- a/sqlc.yaml +++ b/sqlc.yaml @@ -46,6 +46,10 @@ sql: go_type: type: "string" pointer: true + - column: "documents.words" + go_type: + type: "int64" + pointer: true - column: "documents.olid" go_type: type: "string" diff --git a/templates/document.html b/templates/document.html index 3450b69..76bb601 100644 --- a/templates/document.html +++ b/templates/document.html @@ -295,16 +295,56 @@ {{ or .Data.Author "N/A" }}

-
-

Time Read

-

- {{ .Data.TotalTimeMinutes }} Minutes -

+
+
+

Time Read

+ + + +
+
+

Seconds / Page

+

+ {{ .Data.SecondsPerPage }} +

+
+
+

Words / Minute

+

+ {{ .Statistics.WordsPerMinute }} +

+
+
+

Est. Time Left

+

+ {{ NiceSeconds .Statistics.TotalTimeLeftSeconds }} +

+
+
+
+

+ {{ NiceSeconds .Data.TotalTimeSeconds }} +

Progress

- {{ .Data.CurrentPage }} / {{ .Data.TotalPages }} ({{ .Data.Percentage }}%) + {{ .Data.CurrentPage }} / {{ .Data.TotalPages }} ({{ .Data.Percentage }}%)