diff --git a/README.md b/README.md index 5a7e99e..6a96faa 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,7 @@ In additional to the compatible KOSync API's, we add: | DATA_PATH | /data | Directory where to store the documents and cover metadata | | LISTEN_PORT | 8585 | Port the server listens at | | REGISTRATION_ENABLED | false | Whether to allow registration (applies to both WebApp & KOSync API) | +| COOKIE_SESSION_KEY | | Optional secret cookie session key (auto generated if not provided) | # Client (KOReader Plugin) diff --git a/api/api.go b/api/api.go index ba0d93c..78a773f 100644 --- a/api/api.go +++ b/api/api.go @@ -9,31 +9,44 @@ import ( "github.com/gin-contrib/sessions" "github.com/gin-contrib/sessions/cookie" "github.com/gin-gonic/gin" + "github.com/microcosm-cc/bluemonday" + log "github.com/sirupsen/logrus" "reichard.io/bbank/config" "reichard.io/bbank/database" "reichard.io/bbank/graph" ) type API struct { - Router *gin.Engine - Config *config.Config - DB *database.DBManager + Router *gin.Engine + Config *config.Config + DB *database.DBManager + HTMLPolicy *bluemonday.Policy } func NewApi(db *database.DBManager, c *config.Config) *API { api := &API{ - Router: gin.Default(), - Config: c, - DB: db, + HTMLPolicy: bluemonday.StripTagsPolicy(), + Router: gin.Default(), + Config: c, + DB: db, } // Assets & Web App Templates api.Router.Static("/assets", "./assets") // Generate Secure Token - newToken, err := generateToken(64) - if err != nil { - panic("Unable to generate secure token") + var newToken []byte + var err error + + if c.CookieSessionKey != "" { + log.Info("[NewApi] Utilizing Environment Cookie Session Key") + newToken = []byte(c.CookieSessionKey) + } else { + log.Info("[NewApi] Generating Cookie Session Key") + newToken, err = generateToken(64) + if err != nil { + panic("Unable to generate secure token") + } } // Configure Cookie Session Store @@ -69,6 +82,7 @@ func (api *API) registerWebAppRoutes() { render.AddFromFilesFuncs("graphs", helperFuncs, "templates/base.html", "templates/graphs.html") render.AddFromFilesFuncs("activity", helperFuncs, "templates/base.html", "templates/activity.html") render.AddFromFilesFuncs("documents", helperFuncs, "templates/base.html", "templates/documents.html") + render.AddFromFilesFuncs("document", helperFuncs, "templates/base.html", "templates/document.html") api.Router.HTMLRender = render @@ -80,8 +94,9 @@ func (api *API) registerWebAppRoutes() { api.Router.POST("/register", api.authFormRegister) api.Router.GET("/", api.authWebAppMiddleware, api.createAppResourcesRoute("home")) - api.Router.GET("/documents", api.authWebAppMiddleware, api.createAppResourcesRoute("documents")) api.Router.GET("/activity", api.authWebAppMiddleware, api.createAppResourcesRoute("activity")) + api.Router.GET("/documents", api.authWebAppMiddleware, api.createAppResourcesRoute("documents")) + api.Router.GET("/documents/:document", api.authWebAppMiddleware, api.createAppResourcesRoute("document")) api.Router.GET("/documents/:document/file", api.authWebAppMiddleware, api.downloadDocumentFile) api.Router.GET("/documents/:document/cover", api.authWebAppMiddleware, api.getDocumentCover) diff --git a/api/app-routes.go b/api/app-routes.go index 002ec64..3670c97 100644 --- a/api/app-routes.go +++ b/api/app-routes.go @@ -5,6 +5,7 @@ import ( "net/http" "os" "path/filepath" + "time" "github.com/gin-gonic/gin" log "github.com/sirupsen/logrus" @@ -64,12 +65,38 @@ func (api *API) createAppResourcesRoute(routeName string, args ...map[string]any } templateVars["Data"] = documents + } else if routeName == "document" { + var rDocID requestDocumentID + if err := c.ShouldBindUri(&rDocID); err != nil { + log.Error("[createAppResourcesRoute] Invalid URI Bind") + c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Request"}) + return + } + + document, err := api.DB.Queries.GetDocumentWithStats(api.DB.Ctx, database.GetDocumentWithStatsParams{ + UserID: rUser.(string), + DocumentID: rDocID.DocumentID, + }) + if err != nil { + log.Error("[createAppResourcesRoute] GetDocumentWithStats DB Error:", err) + c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Request"}) + return + } + + templateVars["Data"] = document } else if routeName == "activity" { - activity, err := api.DB.Queries.GetActivity(api.DB.Ctx, database.GetActivityParams{ + activityFilter := database.GetActivityParams{ UserID: rUser.(string), Offset: (*qParams.Page - 1) * *qParams.Limit, Limit: *qParams.Limit, - }) + } + + if qParams.Document != nil { + activityFilter.DocFilter = true + activityFilter.DocumentID = *qParams.Document + } + + activity, err := api.DB.Queries.GetActivity(api.DB.Ctx, activityFilter) if err != nil { log.Error("[createAppResourcesRoute] GetActivity DB Error:", err) c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Request"}) @@ -78,6 +105,7 @@ func (api *API) createAppResourcesRoute(routeName string, args ...map[string]any templateVars["Data"] = activity } else if routeName == "home" { + start_time := time.Now() weekly_streak, err := api.DB.Queries.GetUserWindowStreaks(api.DB.Ctx, database.GetUserWindowStreaksParams{ UserID: rUser.(string), Window: "WEEK", @@ -85,6 +113,8 @@ func (api *API) createAppResourcesRoute(routeName string, args ...map[string]any if err != nil { log.Warn("[createAppResourcesRoute] GetUserWindowStreaks DB Error:", err) } + log.Info("GetUserWindowStreaks - WEEK - ", time.Since(start_time)) + start_time = time.Now() daily_streak, err := api.DB.Queries.GetUserWindowStreaks(api.DB.Ctx, database.GetUserWindowStreaksParams{ UserID: rUser.(string), @@ -93,9 +123,15 @@ func (api *API) createAppResourcesRoute(routeName string, args ...map[string]any if err != nil { log.Warn("[createAppResourcesRoute] GetUserWindowStreaks DB Error:", err) } + log.Info("GetUserWindowStreaks - DAY - ", time.Since(start_time)) + start_time = time.Now() database_info, _ := api.DB.Queries.GetDatabaseInfo(api.DB.Ctx, rUser.(string)) + log.Info("GetDatabaseInfo - ", time.Since(start_time)) + + start_time = time.Now() read_graph_data, _ := api.DB.Queries.GetDailyReadStats(api.DB.Ctx, rUser.(string)) + log.Info("GetDailyReadStats - ", time.Since(start_time)) templateVars["Data"] = gin.H{ "DailyStreak": daily_streak, @@ -156,17 +192,39 @@ func (api *API) getDocumentCover(c *gin.Context) { */ var coverID string = "UNKNOWN" - var coverFilePath *string + var coverFilePath string // Identify Documents & Save Covers - coverIDs, err := metadata.GetCoverIDs(document.Title, document.Author) - if err == nil && len(coverIDs) > 0 { - coverFilePath, err = metadata.DownloadAndSaveCover(coverIDs[0], api.Config.DataPath) + bookMetadata := metadata.MetadataInfo{ + Title: document.Title, + Author: document.Author, + } + err = metadata.GetMetadata(&bookMetadata) + if err == nil && bookMetadata.GBID != nil { + // Derive & Sanitize File Name + fileName := "." + filepath.Clean(fmt.Sprintf("/%s.jpg", *bookMetadata.GBID)) + + // Generate Storage Path + coverFilePath = filepath.Join(api.Config.DataPath, "covers", fileName) + + err := metadata.SaveCover(*bookMetadata.GBID, coverFilePath) if err == nil { - coverID = coverIDs[0] + coverID = *bookMetadata.GBID + log.Info("Title:", *bookMetadata.Title) + log.Info("Author:", *bookMetadata.Author) + log.Info("Description:", *bookMetadata.Description) + log.Info("IDs:", bookMetadata.ISBN) } } + // coverIDs, err := metadata.GetCoverOLIDs(document.Title, document.Author) + // if err == nil && len(coverIDs) > 0 { + // coverFilePath, err = metadata.DownloadAndSaveCover(coverIDs[0], api.Config.DataPath) + // if err == nil { + // coverID = coverIDs[0] + // } + // } + // Upsert Document if _, err = api.DB.Queries.UpsertDocument(api.DB.Ctx, database.UpsertDocumentParams{ ID: document.ID, @@ -181,5 +239,5 @@ func (api *API) getDocumentCover(c *gin.Context) { return } - c.File(*coverFilePath) + c.File(coverFilePath) } diff --git a/api/ko-routes.go b/api/ko-routes.go index d4a68cd..df16738 100644 --- a/api/ko-routes.go +++ b/api/ko-routes.go @@ -335,12 +335,12 @@ func (api *API) addDocuments(c *gin.Context) { for _, doc := range rNewDocs.Documents { doc, err := qtx.UpsertDocument(api.DB.Ctx, database.UpsertDocumentParams{ ID: doc.ID, - Title: doc.Title, - Author: doc.Author, - Series: doc.Series, + Title: api.sanitizeInput(doc.Title), + Author: api.sanitizeInput(doc.Author), + Series: api.sanitizeInput(doc.Series), SeriesIndex: doc.SeriesIndex, - Lang: doc.Lang, - Description: doc.Description, + Lang: api.sanitizeInput(doc.Lang), + Description: api.sanitizeInput(doc.Description), }) if err != nil { log.Error("[addDocuments] UpsertDocument DB Error:", err) @@ -605,6 +605,22 @@ func (api *API) downloadDocumentFile(c *gin.Context) { c.File(filePath) } +func (api *API) sanitizeInput(val any) *string { + switch v := val.(type) { + case *string: + if v != nil { + newString := api.HTMLPolicy.Sanitize(string(*v)) + return &newString + } + case string: + if v != "" { + newString := api.HTMLPolicy.Sanitize(string(v)) + return &newString + } + } + return nil +} + func getKeys[M ~map[K]V, K comparable, V any](m M) []K { r := make([]K, 0, len(m)) for k := range m { diff --git a/assets/book55.jpg b/assets/book55.jpg deleted file mode 100644 index 5505e5e..0000000 Binary files a/assets/book55.jpg and /dev/null differ diff --git a/config/config.go b/config/config.go index 2a51e93..f0c7ea6 100644 --- a/config/config.go +++ b/config/config.go @@ -21,6 +21,7 @@ type Config struct { // Miscellaneous Settings RegistrationEnabled bool + CookieSessionKey string } func Load() *Config { @@ -32,6 +33,7 @@ func Load() *Config { ConfigPath: getEnv("CONFIG_PATH", "/config"), DataPath: getEnv("DATA_PATH", "/data"), ListenPort: getEnv("LISTEN_PORT", "8585"), + CookieSessionKey: trimLowerString(getEnv("COOKIE_SESSION_KEY", "")), RegistrationEnabled: trimLowerString(getEnv("REGISTRATION_ENABLED", "false")) == "true", } } diff --git a/database/query.sql b/database/query.sql index ef92c4c..3bd8ad9 100644 --- a/database/query.sql +++ b/database/query.sql @@ -154,6 +154,43 @@ ORDER BY created_at DESC LIMIT $limit OFFSET $offset; +-- name: GetDocumentWithStats :one +WITH true_progress AS ( + SELECT + start_time AS last_read, + SUM(duration) / 60 AS total_time_minutes, + document_id, + current_page, + total_pages, + ROUND(CAST(current_page AS REAL) / CAST(total_pages AS REAL) * 100, 2) AS percentage + FROM activity + WHERE user_id = $user_id + AND document_id = $document_id + GROUP BY document_id + HAVING MAX(start_time) + LIMIT 1 +) +SELECT + documents.*, + + CAST(IFNULL(current_page, 0) AS INTEGER) AS current_page, + CAST(IFNULL(total_pages, 0) AS INTEGER) AS total_pages, + CAST(IFNULL(total_time_minutes, 0) AS INTEGER) AS total_time_minutes, + CAST(DATETIME(IFNULL(last_read, "1970-01-01"), time_offset) AS TEXT) AS last_read, + + CAST(CASE + WHEN percentage > 97.0 THEN 100.0 + WHEN percentage IS NULL THEN 0.0 + ELSE percentage + END AS REAL) AS percentage + +FROM documents +LEFT JOIN true_progress ON true_progress.document_id = documents.id +LEFT JOIN users ON users.id = $user_id +WHERE documents.id = $document_id +ORDER BY true_progress.last_read DESC, documents.created_at DESC +LIMIT 1; + -- name: GetDocumentsWithStats :many WITH true_progress AS ( SELECT @@ -272,7 +309,7 @@ FROM document_days; WITH document_windows AS ( SELECT CASE - WHEN ?2 = "WEEK" THEN STRFTIME('%Y-%m-%d', start_time, time_offset, 'weekday 0', '-7 day') + WHEN ?2 = "WEEK" THEN DATE(start_time, time_offset, 'weekday 0', '-7 day') WHEN ?2 = "DAY" THEN DATE(start_time, time_offset) END AS read_window, time_offset @@ -281,7 +318,6 @@ WITH document_windows AS ( WHERE user_id = $user_id AND CAST($window AS TEXT) = CAST($window AS TEXT) GROUP BY read_window - ORDER BY read_window DESC ), partitions AS ( SELECT @@ -312,6 +348,7 @@ max_streak AS ( start_date AS max_streak_start_date, end_date AS max_streak_end_date FROM streaks + LIMIT 1 ), current_streak AS ( SELECT @@ -320,7 +357,7 @@ current_streak AS ( end_date AS current_streak_end_date FROM streaks WHERE CASE - WHEN ?2 = "WEEK" THEN STRFTIME('%Y-%m-%d', 'now', time_offset, 'weekday 0', '-7 day') = current_streak_end_date + WHEN ?2 = "WEEK" THEN DATE('now', time_offset, 'weekday 0', '-7 day') = current_streak_end_date WHEN ?2 = "DAY" THEN DATE('now', time_offset, '-1 day') = current_streak_end_date OR DATE('now', time_offset) = current_streak_end_date END LIMIT 1 @@ -360,6 +397,7 @@ activity_records AS ( FROM activity LEFT JOIN users ON users.id = activity.user_id WHERE user_id = $user_id + AND start_time > DATE('now', '-31 days') GROUP BY day ORDER BY day DESC LIMIT 30 diff --git a/database/query.sql.go b/database/query.sql.go index 6848458..1795fdf 100644 --- a/database/query.sql.go +++ b/database/query.sql.go @@ -190,6 +190,7 @@ activity_records AS ( FROM activity LEFT JOIN users ON users.id = activity.user_id WHERE user_id = ?1 + AND start_time > DATE('now', '-31 days') GROUP BY day ORDER BY day DESC LIMIT 30 @@ -483,6 +484,98 @@ func (q *Queries) GetDocumentReadStatsCapped(ctx context.Context, arg GetDocumen return i, err } +const getDocumentWithStats = `-- name: GetDocumentWithStats :one +WITH true_progress AS ( + SELECT + start_time AS last_read, + SUM(duration) / 60 AS total_time_minutes, + document_id, + current_page, + total_pages, + ROUND(CAST(current_page AS REAL) / CAST(total_pages AS REAL) * 100, 2) AS percentage + FROM activity + WHERE user_id = ?1 + AND document_id = ?2 + GROUP BY document_id + HAVING MAX(start_time) + LIMIT 1 +) +SELECT + documents.id, documents.md5, documents.filepath, documents.title, documents.author, documents.series, documents.series_index, documents.lang, documents.description, documents.olid, documents.synced, documents.deleted, documents.updated_at, documents.created_at, + + CAST(IFNULL(current_page, 0) AS INTEGER) AS current_page, + CAST(IFNULL(total_pages, 0) AS INTEGER) AS total_pages, + CAST(IFNULL(total_time_minutes, 0) AS INTEGER) AS total_time_minutes, + CAST(DATETIME(IFNULL(last_read, "1970-01-01"), time_offset) AS TEXT) AS last_read, + + CAST(CASE + WHEN percentage > 97.0 THEN 100.0 + WHEN percentage IS NULL THEN 0.0 + ELSE percentage + END AS REAL) AS percentage + +FROM documents +LEFT JOIN true_progress ON true_progress.document_id = documents.id +LEFT JOIN users ON users.id = ?1 +WHERE documents.id = ?2 +ORDER BY true_progress.last_read DESC, documents.created_at DESC +LIMIT 1 +` + +type GetDocumentWithStatsParams struct { + UserID string `json:"user_id"` + DocumentID string `json:"document_id"` +} + +type GetDocumentWithStatsRow struct { + ID string `json:"id"` + Md5 *string `json:"md5"` + Filepath *string `json:"filepath"` + Title *string `json:"title"` + Author *string `json:"author"` + Series *string `json:"series"` + SeriesIndex *int64 `json:"series_index"` + Lang *string `json:"lang"` + Description *string `json:"description"` + Olid *string `json:"-"` + Synced bool `json:"-"` + Deleted bool `json:"-"` + UpdatedAt time.Time `json:"updated_at"` + CreatedAt time.Time `json:"created_at"` + CurrentPage int64 `json:"current_page"` + TotalPages int64 `json:"total_pages"` + TotalTimeMinutes int64 `json:"total_time_minutes"` + LastRead string `json:"last_read"` + Percentage float64 `json:"percentage"` +} + +func (q *Queries) GetDocumentWithStats(ctx context.Context, arg GetDocumentWithStatsParams) (GetDocumentWithStatsRow, error) { + row := q.db.QueryRowContext(ctx, getDocumentWithStats, arg.UserID, arg.DocumentID) + var i GetDocumentWithStatsRow + err := row.Scan( + &i.ID, + &i.Md5, + &i.Filepath, + &i.Title, + &i.Author, + &i.Series, + &i.SeriesIndex, + &i.Lang, + &i.Description, + &i.Olid, + &i.Synced, + &i.Deleted, + &i.UpdatedAt, + &i.CreatedAt, + &i.CurrentPage, + &i.TotalPages, + &i.TotalTimeMinutes, + &i.LastRead, + &i.Percentage, + ) + return i, err +} + const getDocuments = `-- name: GetDocuments :many SELECT id, md5, filepath, title, author, series, series_index, lang, description, olid, synced, deleted, updated_at, created_at FROM documents ORDER BY created_at DESC @@ -783,7 +876,7 @@ const getUserWindowStreaks = `-- name: GetUserWindowStreaks :one WITH document_windows AS ( SELECT CASE - WHEN ?2 = "WEEK" THEN STRFTIME('%Y-%m-%d', start_time, time_offset, 'weekday 0', '-7 day') + WHEN ?2 = "WEEK" THEN DATE(start_time, time_offset, 'weekday 0', '-7 day') WHEN ?2 = "DAY" THEN DATE(start_time, time_offset) END AS read_window, time_offset @@ -792,7 +885,6 @@ WITH document_windows AS ( WHERE user_id = ?1 AND CAST(?2 AS TEXT) = CAST(?2 AS TEXT) GROUP BY read_window - ORDER BY read_window DESC ), partitions AS ( SELECT @@ -823,6 +915,7 @@ max_streak AS ( start_date AS max_streak_start_date, end_date AS max_streak_end_date FROM streaks + LIMIT 1 ), current_streak AS ( SELECT @@ -831,7 +924,7 @@ current_streak AS ( end_date AS current_streak_end_date FROM streaks WHERE CASE - WHEN ?2 = "WEEK" THEN STRFTIME('%Y-%m-%d', 'now', time_offset, 'weekday 0', '-7 day') = current_streak_end_date + WHEN ?2 = "WEEK" THEN DATE('now', time_offset, 'weekday 0', '-7 day') = current_streak_end_date WHEN ?2 = "DAY" THEN DATE('now', time_offset, '-1 day') = current_streak_end_date OR DATE('now', time_offset) = current_streak_end_date END LIMIT 1 diff --git a/database/schema.sql b/database/schema.sql index ebff825..84679fa 100644 --- a/database/schema.sql +++ b/database/schema.sql @@ -1,4 +1,5 @@ PRAGMA foreign_keys = ON; +PRAGMA journal_mode = WAL; -- Authentication CREATE TABLE IF NOT EXISTS users ( diff --git a/go.mod b/go.mod index 5295484..34748da 100644 --- a/go.mod +++ b/go.mod @@ -9,12 +9,14 @@ require ( github.com/gin-contrib/sessions v0.0.4 github.com/gin-gonic/gin v1.9.1 github.com/mattn/go-sqlite3 v1.14.17 + github.com/microcosm-cc/bluemonday v1.0.25 github.com/sirupsen/logrus v1.9.3 github.com/urfave/cli/v2 v2.25.7 golang.org/x/exp v0.0.0-20230905200255-921286631fa9 ) require ( + github.com/aymerick/douceur v0.2.0 // indirect github.com/bytedance/sonic v1.10.0 // indirect github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect github.com/chenzhuoyu/iasm v0.9.0 // indirect @@ -25,6 +27,7 @@ require ( github.com/go-playground/validator/v10 v10.15.3 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/gorilla/context v1.1.1 // indirect + github.com/gorilla/css v1.0.0 // indirect github.com/gorilla/securecookie v1.1.1 // indirect github.com/gorilla/sessions v1.2.1 // indirect github.com/json-iterator/go v1.1.12 // indirect diff --git a/go.sum b/go.sum index 22cccd5..63ee546 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,8 @@ github.com/alexedwards/argon2id v0.0.0-20230305115115-4b3c3280a736 h1:qZaEtLxnqY5mJ0fVKbk31NVhlgi0yrKm51Pq/I5wcz4= github.com/alexedwards/argon2id v0.0.0-20230305115115-4b3c3280a736/go.mod h1:mTeFRcTdnpzOlRjMoFYC/80HwVUreupyAiqPkCZQOXc= github.com/antonlindstrom/pgstore v0.0.0-20200229204646-b08ebf1105e0/go.mod h1:2Ti6VUHVxpC0VSmTZzEvpzysnaGAfGBOoMIz5ykPyyw= +github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= +github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/boj/redistore v0.0.0-20180917114910-cd5dcc76aeff/go.mod h1:+RTT1BOk5P97fT2CiHkbFQwkK3mjsFAP6zCYV2aXtjw= github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b/go.mod h1:H0wQNHz2YrLsuXOZozoeDmnHXkNCRmMW0gwFWDfEZDA= github.com/bradleypeabody/gorilla-sessions-memcache v0.0.0-20181103040241-659414f458e1/go.mod h1:dkChI7Tbtx7H1Tj7TqGSZMOeGpMP5gLHtjroHd4agiI= @@ -58,6 +60,8 @@ github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY= +github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c= github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.1.1/go.mod h1:8KCfur6+4Mqcc6S0FEfKuN15Vl5MgXW92AE8ovaJD0w= @@ -92,6 +96,8 @@ github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM= github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/memcachier/mc v2.0.1+incompatible/go.mod h1:7bkvFE61leUBvXz+yxsOnGBQSZpBSPIMUQSmmSHvuXc= +github.com/microcosm-cc/bluemonday v1.0.25 h1:4NEwSfiJ+Wva0VxN5B8OwMicaJvD8r9tlJWm9rtloEg= +github.com/microcosm-cc/bluemonday v1.0.25/go.mod h1:ZIOjCQp1OrzBBPIJmfX4qDYFuhU02nx4bn030ixfHLE= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= diff --git a/main.go b/main.go index b529da2..4e2e2b0 100644 --- a/main.go +++ b/main.go @@ -48,7 +48,9 @@ func cmdServer(ctx *cli.Context) error { signal.Notify(c, os.Interrupt) <-c + log.Info("Stopping Server") server.StopServer() + log.Info("Server Stopped") os.Exit(0) return nil diff --git a/metadata/metadata.go b/metadata/metadata.go index 46d6709..e6f8b68 100644 --- a/metadata/metadata.go +++ b/metadata/metadata.go @@ -8,97 +8,167 @@ import ( "net/http" "net/url" "os" - "path/filepath" log "github.com/sirupsen/logrus" ) -type coverResult struct { - CoverEditionKey string `json:"cover_edition_key"` +type MetadataInfo struct { + Title *string + Author *string + Description *string + GBID *string + ISBN []*string } -type queryResponse struct { - ResultCount int `json:"numFound"` - Start int `json:"start"` - ResultCountExact bool `json:"numFoundExact"` - Results []coverResult `json:"docs"` +type gBooksIdentifiers struct { + Type string `json:"type"` + Identifier string `json:"identifier"` } -var BASE_QUERY_URL string = "https://openlibrary.org/search.json?q=%s&fields=cover_edition_key" -var BASE_COVER_URL string = "https://covers.openlibrary.org/b/olid/%s-L.jpg" +type gBooksInfo struct { + Title string `json:"title"` + Authors []string `json:"authors"` + Description string `json:"description"` + Identifiers []gBooksIdentifiers `json:"industryIdentifiers"` +} -func GetCoverIDs(title *string, author *string) ([]string, error) { - if title == nil || author == nil { - log.Error("[metadata] Invalid Search Query") - return nil, errors.New("Invalid Query") - } +type gBooksQueryItem struct { + ID string `json:"id"` + Info gBooksInfo `json:"volumeInfo"` +} - searchQuery := url.QueryEscape(fmt.Sprintf("%s %s", *title, *author)) - apiQuery := fmt.Sprintf(BASE_QUERY_URL, searchQuery) +type gBooksQueryResponse struct { + TotalItems int `json:"totalItems"` + Items []gBooksQueryItem `json:"items"` +} - log.Info("[metadata] Acquiring CoverID") - resp, err := http.Get(apiQuery) - if err != nil { - log.Error("[metadata] Cover URL API Failure") - return nil, errors.New("API Failure") - } +const GBOOKS_QUERY_URL string = "https://www.googleapis.com/books/v1/volumes?q=%s&filter=ebooks&download=epub" +const GBOOKS_GBID_INFO_URL string = "https://www.googleapis.com/books/v1/volumes/%s" +const GBOOKS_GBID_COVER_URL string = "https://books.google.com/books/content/images/frontcover/%s?fife=w480-h690" - target := queryResponse{} - err = json.NewDecoder(resp.Body).Decode(&target) - if err != nil { - log.Error("[metadata] Cover URL API Decode Failure") - return nil, errors.New("API Failure") - } - - var coverIDs []string - for _, result := range target.Results { - if result.CoverEditionKey != "" { - coverIDs = append(coverIDs, result.CoverEditionKey) +func GetMetadata(data *MetadataInfo) error { + var queryResult *gBooksQueryItem + if data.GBID != nil { + // Use GBID + resp, err := performGBIDRequest(*data.GBID) + if err != nil { + return err } + queryResult = resp + } else if len(data.ISBN) > 0 { + searchQuery := "isbn:" + *data.ISBN[0] + resp, err := performSearchRequest(searchQuery) + if err != nil { + return err + } + queryResult = &resp.Items[0] + } else if data.Title != nil && data.Author != nil { + searchQuery := url.QueryEscape(fmt.Sprintf("%s %s", *data.Title, *data.Author)) + resp, err := performSearchRequest(searchQuery) + if err != nil { + return err + } + queryResult = &resp.Items[0] + } else { + return errors.New("Invalid Data") } - return coverIDs, nil + // Merge Data + data.GBID = &queryResult.ID + data.Description = &queryResult.Info.Description + data.Title = &queryResult.Info.Title + if len(queryResult.Info.Authors) > 0 { + data.Author = &queryResult.Info.Authors[0] + } + for _, item := range queryResult.Info.Identifiers { + if item.Type == "ISBN_10" || item.Type == "ISBN_13" { + data.ISBN = append(data.ISBN, &item.Identifier) + } + + } + + return nil } -func DownloadAndSaveCover(coverID string, dirPath string) (*string, error) { - // Derive & Sanitize File Name - fileName := "." + filepath.Clean(fmt.Sprintf("/%s.jpg", coverID)) - - // Generate Storage Path - safePath := filepath.Join(dirPath, "covers", fileName) - +func SaveCover(id string, safePath string) error { // Validate File Doesn't Exists _, err := os.Stat(safePath) if err == nil { - log.Warn("[metadata] File Alreads Exists") - return &safePath, nil + log.Warn("[SaveCover] File Alreads Exists") + return nil } // Create File out, err := os.Create(safePath) if err != nil { - log.Error("[metadata] File Create Error") - return nil, errors.New("File Failure") + log.Error("[SaveCover] File Create Error") + return errors.New("File Failure") } defer out.Close() // Download File - log.Info("[metadata] Downloading Cover") - coverURL := fmt.Sprintf(BASE_COVER_URL, coverID) + log.Info("[SaveCover] Downloading Cover") + coverURL := fmt.Sprintf(GBOOKS_GBID_COVER_URL, id) resp, err := http.Get(coverURL) if err != nil { - log.Error("[metadata] Cover URL API Failure") - return nil, errors.New("API Failure") + log.Error("[SaveCover] Cover URL API Failure") + return errors.New("API Failure") } defer resp.Body.Close() // Copy File to Disk + log.Info("[SaveCover] Saving Cover") _, err = io.Copy(out, resp.Body) if err != nil { - log.Error("[metadata] File Copy Error") - return nil, errors.New("File Failure") + log.Error("[SaveCover] File Copy Error") + return errors.New("File Failure") } // Return FilePath - return &safePath, nil + return nil +} + +func performSearchRequest(searchQuery string) (*gBooksQueryResponse, error) { + apiQuery := fmt.Sprintf(GBOOKS_QUERY_URL, searchQuery) + + log.Info("[performSearchRequest] Acquiring CoverID") + resp, err := http.Get(apiQuery) + if err != nil { + log.Error("[performSearchRequest] Cover URL API Failure") + return nil, errors.New("API Failure") + } + + parsedResp := gBooksQueryResponse{} + err = json.NewDecoder(resp.Body).Decode(&parsedResp) + if err != nil { + log.Error("[performSearchRequest] Google Books Query API Decode Failure") + return nil, errors.New("API Failure") + } + + if len(parsedResp.Items) == 0 { + log.Warn("[performSearchRequest] No Results") + return nil, errors.New("No Results") + } + + return &parsedResp, nil +} + +func performGBIDRequest(id string) (*gBooksQueryItem, error) { + apiQuery := fmt.Sprintf(GBOOKS_GBID_INFO_URL, id) + + log.Info("[performGBIDRequest] Acquiring CoverID") + resp, err := http.Get(apiQuery) + if err != nil { + log.Error("[performGBIDRequest] Cover URL API Failure") + return nil, errors.New("API Failure") + } + + parsedResp := gBooksQueryItem{} + err = json.NewDecoder(resp.Body).Decode(&parsedResp) + if err != nil { + log.Error("[performGBIDRequest] Google Books ID API Decode Failure") + return nil, errors.New("API Failure") + } + + return &parsedResp, nil } diff --git a/metadata/olib.go b/metadata/olib.go new file mode 100644 index 0000000..0ceeaa6 --- /dev/null +++ b/metadata/olib.go @@ -0,0 +1,107 @@ +package metadata + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + + log "github.com/sirupsen/logrus" +) + +type oLibCoverResult struct { + CoverEditionKey string `json:"cover_edition_key"` +} + +type oLibQueryResponse struct { + ResultCount int `json:"numFound"` + Start int `json:"start"` + ResultCountExact bool `json:"numFoundExact"` + Results []oLibCoverResult `json:"docs"` +} + +const OLIB_QUERY_URL string = "https://openlibrary.org/search.json?q=%s&fields=cover_edition_key" +const OLIB_OLID_COVER_URL string = "https://covers.openlibrary.org/b/olid/%s-L.jpg" +const OLIB_ISBN_COVER_URL string = "https://covers.openlibrary.org/b/isbn/%s-L.jpg" +const OLIB_OLID_LINK_URL string = "https://openlibrary.org/books/%s" +const OLIB_ISBN_LINK_URL string = "https://openlibrary.org/isbn/%s" + +func GetCoverOLIDs(title *string, author *string) ([]string, error) { + if title == nil || author == nil { + log.Error("[metadata] Invalid Search Query") + return nil, errors.New("Invalid Query") + } + + searchQuery := url.QueryEscape(fmt.Sprintf("%s %s", *title, *author)) + apiQuery := fmt.Sprintf(OLIB_QUERY_URL, searchQuery) + + log.Info("[metadata] Acquiring CoverID") + resp, err := http.Get(apiQuery) + if err != nil { + log.Error("[metadata] Cover URL API Failure") + return nil, errors.New("API Failure") + } + + target := oLibQueryResponse{} + err = json.NewDecoder(resp.Body).Decode(&target) + if err != nil { + log.Error("[metadata] Cover URL API Decode Failure") + return nil, errors.New("API Failure") + } + + var coverIDs []string + for _, result := range target.Results { + if result.CoverEditionKey != "" { + coverIDs = append(coverIDs, result.CoverEditionKey) + } + } + + return coverIDs, nil +} + +func DownloadAndSaveCover(coverID string, dirPath string) (*string, error) { + // Derive & Sanitize File Name + fileName := "." + filepath.Clean(fmt.Sprintf("/%s.jpg", coverID)) + + // Generate Storage Path + safePath := filepath.Join(dirPath, "covers", fileName) + + // Validate File Doesn't Exists + _, err := os.Stat(safePath) + if err == nil { + log.Warn("[metadata] File Alreads Exists") + return &safePath, nil + } + + // Create File + out, err := os.Create(safePath) + if err != nil { + log.Error("[metadata] File Create Error") + return nil, errors.New("File Failure") + } + defer out.Close() + + // Download File + log.Info("[metadata] Downloading Cover") + coverURL := fmt.Sprintf(OLIB_OLID_COVER_URL, coverID) + resp, err := http.Get(coverURL) + if err != nil { + log.Error("[metadata] Cover URL API Failure") + return nil, errors.New("API Failure") + } + defer resp.Body.Close() + + // Copy File to Disk + _, err = io.Copy(out, resp.Body) + if err != nil { + log.Error("[metadata] File Copy Error") + return nil, errors.New("File Failure") + } + + // Return FilePath + return &safePath, nil +} diff --git a/server/server.go b/server/server.go index a927277..2124bf9 100644 --- a/server/server.go +++ b/server/server.go @@ -59,4 +59,5 @@ func (s *Server) StopServer() { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() s.httpServer.Shutdown(ctx) + s.API.DB.DB.Close() } diff --git a/templates/activity.html b/templates/activity.html index b1dbb3e..49d1a1c 100644 --- a/templates/activity.html +++ b/templates/activity.html @@ -1,9 +1,9 @@ -{{template "base.html" .}} {{define "title"}}Activity{{end}} {{define -"content"}} - +{{template "base.html" .}} {{define "title"}}Activity{{end}} {{define "header"}} +Activity +{{end}} {{define "content"}}
- +
-

{{ $activity.Author }} - {{ $activity.Title }}

+ {{ $activity.Author }} - {{ $activity.Title }}

{{ $activity.StartTime }}

diff --git a/templates/base.html b/templates/base.html index 40e5e78..e9041b4 100644 --- a/templates/base.html +++ b/templates/base.html @@ -5,7 +5,7 @@ - {{block "title" .}}{{end}} + Book Manager - {{block "title" .}}{{end}}
-

{{block "title" .}}{{end}}

+

{{block "header" .}}{{end}}

diff --git a/templates/base.old.html b/templates/base.old.html deleted file mode 100644 index f5b5bfa..0000000 --- a/templates/base.old.html +++ /dev/null @@ -1,182 +0,0 @@ - - - - - - - {{block "title" .}}{{end}} - - - - - - -
-
- {{block "content" .}}{{end}} -
-
- - diff --git a/templates/document.html b/templates/document.html new file mode 100644 index 0000000..d90ac70 --- /dev/null +++ b/templates/document.html @@ -0,0 +1,161 @@ +{{template "base.html" .}} + +{{define "title"}}Documents{{end}} + +{{define "header"}} +Documents +{{end}} + +{{define "content"}} +
+
+ +
+
+ + +
+

Are you sure?

+ +
+
+ + + + + + +
+ + +
+ +
+ +
+
+ {{ if .Data.Filepath }} + + + + + + {{ else }} + + + + {{ end }} +
+
+
+
+

Title

+ +

+ {{ or .Data.Title "N/A" }} +

+
+
+

Author

+

+ {{ or .Data.Author "N/A" }} +

+
+
+

Progress

+

+ {{ .Data.CurrentPage }} / {{ .Data.TotalPages }} ({{ .Data.Percentage }}%) +

+
+
+

Minutes Read

+

+ {{ .Data.TotalTimeMinutes }} Minutes +

+
+
+

Description

+

+ {{ or .Data.Description "N/A" }} +

+
+ +{{end}} diff --git a/templates/documents.html b/templates/documents.html index 3483f8d..1544a56 100644 --- a/templates/documents.html +++ b/templates/documents.html @@ -1,15 +1,22 @@ -{{template "base.html" .}} {{define "title"}}Documents{{end}} {{define -"content"}} +{{template "base.html" .}} + +{{define "title"}}Documents{{end}} + +{{define "header"}} +Documents +{{end}} + +{{define "content"}}
{{range $doc := .Data }} -
+
- +
-
+

Title

@@ -43,9 +50,57 @@
+
+ + + + + + + + {{ if $doc.Filepath }} + + + + + + {{ else }} + + + + {{ end }} +
- {{end}}
{{end}} diff --git a/templates/graphs.html b/templates/graphs.html index 0b118e9..7c737ce 100644 --- a/templates/graphs.html +++ b/templates/graphs.html @@ -1,3 +1,11 @@ -{{template "base.html" .}} {{define "title"}}Graphs{{end}} {{define "content"}} +{{template "base.html" .}} + +{{define "title"}}Graphs{{end}} + +{{define "header"}} +Graphs +{{end}} + +{{define "content"}}

Graphs

{{end}} diff --git a/templates/header.html b/templates/header.html deleted file mode 100644 index a314344..0000000 --- a/templates/header.html +++ /dev/null @@ -1,162 +0,0 @@ - - - - diff --git a/templates/home.html b/templates/home.html index d085ab3..b659e7a 100644 --- a/templates/home.html +++ b/templates/home.html @@ -1,5 +1,6 @@ -{{template "base.html" .}} {{define "title"}}Home{{end}} {{define "content"}} - +{{template "base.html" .}} {{define "title"}}Home{{end}} {{define "header"}} +Home +{{end}} {{define "content"}}
-
+

{{ .Data.DatabaseInfo.DocumentsSize }}

@@ -116,7 +117,7 @@
-
+

{{ .Data.DatabaseInfo.ActivitySize }}

@@ -129,7 +130,7 @@
-
+

{{ .Data.DatabaseInfo.ProgressSize }}

@@ -142,7 +143,7 @@
-
+

{{ .Data.DatabaseInfo.DevicesSize }}

@@ -173,7 +174,7 @@ >

Current Daily Streak

-
+
{{ .Data.DailyStreak.CurrentStreakStartDate }} ➞ {{ .Data.DailyStreak.CurrentStreakEndDate }}
@@ -185,7 +186,7 @@

Best Daily Streak

-
+
{{ .Data.DailyStreak.MaxStreakStartDate }} ➞ {{ .Data.DailyStreak.MaxStreakEndDate }}
@@ -218,7 +219,7 @@ >

Current Weekly Streak

-
+
{{ .Data.WeeklyStreak.CurrentStreakStartDate }} ➞ {{ .Data.WeeklyStreak.CurrentStreakEndDate }}
@@ -230,7 +231,7 @@

Best Weekly Streak

-
+
{{ .Data.WeeklyStreak.MaxStreakStartDate }} ➞ {{ .Data.WeeklyStreak.MaxStreakEndDate }}
diff --git a/templates/login.html b/templates/login.html index 045df2e..71e0dbb 100644 --- a/templates/login.html +++ b/templates/login.html @@ -16,8 +16,8 @@