From f2163c8fd9bd4c9ae066d6261cdb5d2f3683feb3 Mon Sep 17 00:00:00 2001 From: Evan Reichard Date: Wed, 20 Sep 2023 20:35:01 -0400 Subject: [PATCH] [fix] map concurrency issue, [add] better logging, [add] activity template, [fix] safari redirect issue, [add] timezone framework --- api/api.go | 3 +- api/app-routes.go | 56 +++++++++++----- api/auth.go | 2 + api/ko-routes.go | 112 ++++++++++++++++++-------------- api/web-routes.go | 2 +- database/models.go | 9 +-- database/query.sql | 65 ++++++++++--------- database/query.sql.go | 105 ++++++++++++++++++------------ database/schema.sql | 1 + templates/activity.html | 54 +++++++++++++++- templates/base.html | 136 ++++++++++++++++++++++++--------------- templates/documents.html | 4 +- templates/home.html | 21 ++++-- 13 files changed, 368 insertions(+), 202 deletions(-) diff --git a/api/api.go b/api/api.go index 5e20776..f8716dc 100644 --- a/api/api.go +++ b/api/api.go @@ -80,13 +80,12 @@ func (api *API) registerWebAppRoutes() { api.Router.GET("/", api.authWebAppMiddleware, api.createAppResourcesRoute("home")) api.Router.GET("/documents", api.authWebAppMiddleware, api.createAppResourcesRoute("documents")) + api.Router.GET("/activity", api.authWebAppMiddleware, api.createAppResourcesRoute("activity")) api.Router.GET("/documents/:document/file", api.authWebAppMiddleware, api.downloadDocumentFile) api.Router.GET("/documents/:document/cover", api.authWebAppMiddleware, api.getDocumentCover) // TODO - api.Router.GET("/activity", api.authWebAppMiddleware, baseResourceRoute("activity")) api.Router.GET("/graphs", api.authWebAppMiddleware, baseResourceRoute("graphs")) - } func (api *API) registerKOAPIRoutes(apiGroup *gin.RouterGroup) { diff --git a/api/app-routes.go b/api/app-routes.go index 1cb49fe..be3f4d8 100644 --- a/api/app-routes.go +++ b/api/app-routes.go @@ -27,17 +27,25 @@ func baseResourceRoute(template string, args ...map[string]any) func(c *gin.Cont func (api *API) createAppResourcesRoute(routeName string, args ...map[string]any) func(*gin.Context) { // Merge Optional Template Data - var templateVars = gin.H{} + var templateVarsBase = gin.H{} if len(args) > 0 { - templateVars = args[0] + templateVarsBase = args[0] } - templateVars["RouteName"] = routeName + templateVarsBase["RouteName"] = routeName return func(c *gin.Context) { rUser, _ := c.Get("AuthorizedUser") - qParams := bindQueryParams(c) + + // Copy Base & Update + templateVars := gin.H{} + for k, v := range templateVarsBase { + templateVars[k] = v + } templateVars["User"] = rUser + // Potential URL Parameters + qParams := bindQueryParams(c) + if routeName == "documents" { documents, err := api.DB.Queries.GetDocumentsWithStats(api.DB.Ctx, database.GetDocumentsWithStatsParams{ UserID: rUser.(string), @@ -45,28 +53,44 @@ func (api *API) createAppResourcesRoute(routeName string, args ...map[string]any Limit: *qParams.Limit, }) if err != nil { - log.Info(err) + log.Error("[createAppResourcesRoute] GetDocumentsWithStats DB Error:", err) c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Request"}) return } templateVars["Data"] = documents + } else if routeName == "activity" { + activity, err := api.DB.Queries.GetActivity(api.DB.Ctx, database.GetActivityParams{ + UserID: rUser.(string), + Offset: (*qParams.Page - 1) * *qParams.Limit, + Limit: *qParams.Limit, + }) + if err != nil { + log.Error("[createAppResourcesRoute] GetActivity DB Error:", err) + c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Request"}) + return + } + + templateVars["Data"] = activity } else if routeName == "home" { - weekly_streak, _ := api.DB.Queries.GetUserWindowStreaks(api.DB.Ctx, database.GetUserWindowStreaksParams{ + weekly_streak, err := api.DB.Queries.GetUserWindowStreaks(api.DB.Ctx, database.GetUserWindowStreaksParams{ UserID: rUser.(string), Window: "WEEK", }) + if err != nil { + log.Warn("[createAppResourcesRoute] GetUserWindowStreaks DB Error:", err) + } - daily_streak, _ := api.DB.Queries.GetUserWindowStreaks(api.DB.Ctx, database.GetUserWindowStreaksParams{ + daily_streak, err := api.DB.Queries.GetUserWindowStreaks(api.DB.Ctx, database.GetUserWindowStreaksParams{ UserID: rUser.(string), Window: "DAY", }) + if err != nil { + log.Warn("[createAppResourcesRoute] GetUserWindowStreaks DB Error:", err) + } database_info, _ := api.DB.Queries.GetDatabaseInfo(api.DB.Ctx, rUser.(string)) - read_graph_data, err := api.DB.Queries.GetDailyReadStats(api.DB.Ctx, rUser.(string)) - if err != nil { - log.Info("HMMMM:", err) - } + read_graph_data, _ := api.DB.Queries.GetDailyReadStats(api.DB.Ctx, rUser.(string)) templateVars["Data"] = gin.H{ "DailyStreak": daily_streak, @@ -85,6 +109,7 @@ func (api *API) createAppResourcesRoute(routeName string, args ...map[string]any func (api *API) getDocumentCover(c *gin.Context) { var rDoc requestDocumentID if err := c.ShouldBindUri(&rDoc); err != nil { + log.Error("[getDocumentCover] Invalid URI Bind") c.AbortWithStatus(http.StatusBadRequest) return } @@ -92,6 +117,7 @@ func (api *API) getDocumentCover(c *gin.Context) { // Validate Document Exists in DB document, err := api.DB.Queries.GetDocument(api.DB.Ctx, rDoc.DocumentID) if err != nil { + log.Error("[getDocumentCover] GetDocument DB Error:", err) c.AbortWithStatus(http.StatusBadRequest) return } @@ -99,7 +125,7 @@ func (api *API) getDocumentCover(c *gin.Context) { // Handle Identified Document if document.Olid != nil { if *document.Olid == "UNKNOWN" { - c.Redirect(http.StatusFound, "/assets/no-cover.jpg") + c.File("./assets/no-cover.jpg") return } @@ -110,7 +136,7 @@ func (api *API) getDocumentCover(c *gin.Context) { // Validate File Exists _, err = os.Stat(safePath) if err != nil { - c.Redirect(http.StatusFound, "/assets/no-cover.jpg") + c.File("./assets/no-cover.jpg") return } @@ -141,12 +167,12 @@ func (api *API) getDocumentCover(c *gin.Context) { ID: document.ID, Olid: &coverID, }); err != nil { - log.Error("Document Upsert Error") + log.Warn("[getDocumentCover] UpsertDocument DB Error:", err) } // Return Unknown Cover if coverID == "UNKNOWN" { - c.Redirect(http.StatusFound, "/assets/no-cover.jpg") + c.File("./assets/no-cover.jpg") return } diff --git a/api/auth.go b/api/auth.go index 836803c..18815fa 100644 --- a/api/auth.go +++ b/api/auth.go @@ -36,6 +36,7 @@ func (api *API) authAPIMiddleware(c *gin.Context) { // Utilize Session Token if authorizedUser := session.Get("authorizedUser"); authorizedUser != nil { c.Set("AuthorizedUser", authorizedUser) + c.Header("Cache-Control", "private") c.Next() return } @@ -69,6 +70,7 @@ func (api *API) authWebAppMiddleware(c *gin.Context) { // Utilize Session Token if authorizedUser := session.Get("authorizedUser"); authorizedUser != nil { c.Set("AuthorizedUser", authorizedUser) + c.Header("Cache-Control", "private") c.Next() return } diff --git a/api/ko-routes.go b/api/ko-routes.go index d49867e..d4a68cd 100644 --- a/api/ko-routes.go +++ b/api/ko-routes.go @@ -83,21 +83,25 @@ func (api *API) authorizeUser(c *gin.Context) { func (api *API) createUser(c *gin.Context) { if !api.Config.RegistrationEnabled { c.AbortWithStatus(http.StatusConflict) + return } var rUser requestUser if err := c.ShouldBindJSON(&rUser); err != nil { + log.Error("[createUser] Invalid JSON Bind") c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid User Data"}) return } if rUser.Username == "" || rUser.Password == "" { + log.Error("[createUser] Invalid User - Empty Username or Password") c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid User Data"}) return } hashedPassword, err := argon2.CreateHash(rUser.Password, argon2.DefaultParams) if err != nil { + log.Error("[createUser] Argon2 Hash Failure:", err) c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Unknown Error"}) return } @@ -106,20 +110,18 @@ func (api *API) createUser(c *gin.Context) { ID: rUser.Username, Pass: hashedPassword, }) - - // SQL Error if err != nil { + log.Error("[createUser] CreateUser DB Error:", err) c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid User Data"}) return } - // User Exists (ON CONFLICT DO NOTHING) + // User Exists if rows == 0 { c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "User Already Exists"}) return } - // TODO: Struct -> JSON c.JSON(http.StatusCreated, gin.H{ "username": rUser.Username, }) @@ -130,26 +132,25 @@ func (api *API) setProgress(c *gin.Context) { var rPosition requestPosition if err := c.ShouldBindJSON(&rPosition); err != nil { + log.Error("[setProgress] Invalid JSON Bind") c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Progress Data"}) return } // Upsert Device - device, err := api.DB.Queries.UpsertDevice(api.DB.Ctx, database.UpsertDeviceParams{ + if _, err := api.DB.Queries.UpsertDevice(api.DB.Ctx, database.UpsertDeviceParams{ ID: rPosition.DeviceID, UserID: rUser.(string), DeviceName: rPosition.Device, - }) - if err != nil { - log.Error("Device Upsert Error:", device, err) + }); err != nil { + log.Error("[setProgress] UpsertDevice DB Error:", err) } // Upsert Document - document, err := api.DB.Queries.UpsertDocument(api.DB.Ctx, database.UpsertDocumentParams{ + if _, err := api.DB.Queries.UpsertDocument(api.DB.Ctx, database.UpsertDocumentParams{ ID: rPosition.DocumentID, - }) - if err != nil { - log.Error("Document Upsert Error:", document, err) + }); err != nil { + log.Error("[setProgress] UpsertDocument DB Error:", err) } // Create or Replace Progress @@ -161,11 +162,11 @@ func (api *API) setProgress(c *gin.Context) { Progress: rPosition.Progress, }) if err != nil { + log.Error("[setProgress] UpdateProgress DB Error:", err) c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Request"}) return } - // TODO: Struct -> JSON c.JSON(http.StatusOK, gin.H{ "document": progress.DocumentID, "timestamp": progress.CreatedAt, @@ -177,6 +178,7 @@ func (api *API) getProgress(c *gin.Context) { var rDocID requestDocumentID if err := c.ShouldBindUri(&rDocID); err != nil { + log.Error("[getProgress] Invalid URI Bind") c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Request"}) return } @@ -187,12 +189,11 @@ func (api *API) getProgress(c *gin.Context) { }) if err != nil { - log.Error("Invalid Progress:", progress, err) + log.Error("[getProgress] GetProgress DB Error:", err) c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Document"}) return } - // TODO: Struct -> JSON c.JSON(http.StatusOK, gin.H{ "document": progress.DocumentID, "percentage": progress.Percentage, @@ -207,6 +208,7 @@ func (api *API) addActivities(c *gin.Context) { var rActivity requestActivity if err := c.ShouldBindJSON(&rActivity); err != nil { + log.Error("[addActivity] Invalid JSON Bind") c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Activity"}) return } @@ -214,6 +216,7 @@ func (api *API) addActivities(c *gin.Context) { // Do Transaction tx, err := api.DB.DB.Begin() if err != nil { + log.Error("[addActivities] Transaction Begin DB Error:", err) c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Unknown Error"}) return } @@ -231,30 +234,29 @@ func (api *API) addActivities(c *gin.Context) { // Upsert Documents for _, doc := range allDocuments { - _, err := qtx.UpsertDocument(api.DB.Ctx, database.UpsertDocumentParams{ + if _, err := qtx.UpsertDocument(api.DB.Ctx, database.UpsertDocumentParams{ ID: doc, - }) - - if err != nil { + }); err != nil { + log.Error("[addActivities] UpsertDocument DB Error:", err) c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Document"}) return } } // Upsert Device - _, err = qtx.UpsertDevice(api.DB.Ctx, database.UpsertDeviceParams{ + if _, err = qtx.UpsertDevice(api.DB.Ctx, database.UpsertDeviceParams{ ID: rActivity.DeviceID, UserID: rUser.(string), DeviceName: rActivity.Device, - }) - if err != nil { + }); err != nil { + log.Error("[addActivities] UpsertDevice DB Error:", err) c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Device"}) return } // Add All Activity for _, item := range rActivity.Activity { - _, err := qtx.AddActivity(api.DB.Ctx, database.AddActivityParams{ + if _, err := qtx.AddActivity(api.DB.Ctx, database.AddActivityParams{ UserID: rUser.(string), DocumentID: item.DocumentID, DeviceID: rActivity.DeviceID, @@ -262,19 +264,17 @@ func (api *API) addActivities(c *gin.Context) { Duration: int64(item.Duration), CurrentPage: int64(item.CurrentPage), TotalPages: int64(item.TotalPages), - }) - - if err != nil { + }); err != nil { + log.Error("[addActivities] AddActivity DB Error:", err) c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Activity"}) return } } // Commit Transaction - tx.Commit() - - if err != nil { - c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Document"}) + if err := tx.Commit(); err != nil { + log.Error("[addActivities] Transaction Commit DB Error:", err) + c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Unknown Error"}) return } @@ -288,6 +288,7 @@ func (api *API) checkActivitySync(c *gin.Context) { var rCheckActivity requestCheckActivitySync if err := c.ShouldBindJSON(&rCheckActivity); err != nil { + log.Error("[checkActivitySync] Invalid JSON Bind") c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Request"}) return } @@ -300,7 +301,7 @@ func (api *API) checkActivitySync(c *gin.Context) { if err == sql.ErrNoRows { lastActivity = time.UnixMilli(0) } else if err != nil { - log.Error("GetLastActivity Error:", err) + log.Error("[checkActivitySync] GetLastActivity DB Error:", err) c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Unknown Error"}) return } @@ -321,7 +322,7 @@ func (api *API) addDocuments(c *gin.Context) { // Do Transaction tx, err := api.DB.DB.Begin() if err != nil { - log.Error("[addDocuments] Unknown Transaction Error") + log.Error("[addDocuments] Transaction Begin DB Error:", err) c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Unknown Error"}) return } @@ -342,17 +343,16 @@ func (api *API) addDocuments(c *gin.Context) { Description: doc.Description, }) if err != nil { - log.Error("[addDocuments] UpsertDocument Error:", err) + log.Error("[addDocuments] UpsertDocument DB Error:", err) c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Document"}) return } - _, err = qtx.UpdateDocumentSync(api.DB.Ctx, database.UpdateDocumentSyncParams{ + if _, err = qtx.UpdateDocumentSync(api.DB.Ctx, database.UpdateDocumentSyncParams{ ID: doc.ID, Synced: true, - }) - if err != nil { - log.Error("[addDocuments] UpsertDocumentSync Error:", err) + }); err != nil { + log.Error("[addDocuments] UpdateDocumentSync DB Error:", err) c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Document"}) return } @@ -360,7 +360,11 @@ func (api *API) addDocuments(c *gin.Context) { } // Commit Transaction - tx.Commit() + if err := tx.Commit(); err != nil { + log.Error("[addDocuments] Transaction Commit DB Error:", err) + c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Unknown Error"}) + return + } c.JSON(http.StatusOK, gin.H{ "changed": len(rNewDocs.Documents), @@ -372,6 +376,7 @@ func (api *API) checkDocumentsSync(c *gin.Context) { var rCheckDocs requestCheckDocumentSync if err := c.ShouldBindJSON(&rCheckDocs); err != nil { + log.Error("[checkDocumentsSync] Invalid JSON Bind") c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Request"}) return } @@ -383,6 +388,7 @@ func (api *API) checkDocumentsSync(c *gin.Context) { DeviceName: rCheckDocs.Device, }) if err != nil { + log.Error("[checkDocumentsSync] UpsertDevice DB Error", err) c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Device"}) return } @@ -394,7 +400,7 @@ func (api *API) checkDocumentsSync(c *gin.Context) { // Get Missing Documents missingDocs, err = api.DB.Queries.GetMissingDocuments(api.DB.Ctx, rCheckDocs.Have) if err != nil { - log.Error("GetMissingDocuments Error:", err) + log.Error("[checkDocumentsSync] GetMissingDocuments DB Error", err) c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Request"}) return } @@ -402,7 +408,7 @@ func (api *API) checkDocumentsSync(c *gin.Context) { // Get Deleted Documents deletedDocIDs, err = api.DB.Queries.GetDeletedDocuments(api.DB.Ctx, rCheckDocs.Have) if err != nil { - log.Error("GetDeletedDocuements Error:", err) + log.Error("[checkDocumentsSync] GetDeletedDocuments DB Error", err) c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Request"}) return } @@ -411,14 +417,14 @@ func (api *API) checkDocumentsSync(c *gin.Context) { // Get Wanted Documents jsonHaves, err := json.Marshal(rCheckDocs.Have) if err != nil { - log.Error("JSON Marshal Error:", err) + log.Error("[checkDocumentsSync] JSON Marshal Error", err) c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Request"}) return } wantedDocs, err := api.DB.Queries.GetWantedDocuments(api.DB.Ctx, string(jsonHaves)) if err != nil { - log.Error("GetWantedDocuments Error:", err) + log.Error("[checkDocumentsSync] GetWantedDocuments DB Error", err) c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Request"}) return } @@ -462,12 +468,14 @@ func (api *API) checkDocumentsSync(c *gin.Context) { func (api *API) uploadDocumentFile(c *gin.Context) { var rDoc requestDocumentID if err := c.ShouldBindUri(&rDoc); err != nil { + log.Error("[uploadDocumentFile] Invalid URI Bind") c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Request"}) return } fileData, err := c.FormFile("file") if err != nil { + log.Error("[uploadDocumentFile] File Error:", err) c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "File Error"}) return } @@ -478,6 +486,7 @@ func (api *API) uploadDocumentFile(c *gin.Context) { fileExtension := fileMime.Extension() if !slices.Contains(allowedExtensions, fileExtension) { + log.Error("[uploadDocumentFile] Invalid FileType:", fileExtension) c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Filetype"}) return } @@ -485,6 +494,7 @@ func (api *API) uploadDocumentFile(c *gin.Context) { // Validate Document Exists in DB document, err := api.DB.Queries.GetDocument(api.DB.Ctx, rDoc.DocumentID) if err != nil { + log.Error("[uploadDocumentFile] GetDocument DB Error:", err) c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Unknown Document"}) return } @@ -517,6 +527,7 @@ func (api *API) uploadDocumentFile(c *gin.Context) { if os.IsNotExist(err) { err = c.SaveUploadedFile(fileData, safePath) if err != nil { + log.Error("[uploadDocumentFile] Save Failure:", err) c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "File Error"}) return } @@ -525,27 +536,28 @@ func (api *API) uploadDocumentFile(c *gin.Context) { // Get MD5 Hash fileHash, err := getFileMD5(safePath) if err != nil { + log.Error("[uploadDocumentFile] Hash Failure:", err) c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "File Error"}) return } // Upsert Document - _, err = api.DB.Queries.UpsertDocument(api.DB.Ctx, database.UpsertDocumentParams{ + if _, err = api.DB.Queries.UpsertDocument(api.DB.Ctx, database.UpsertDocumentParams{ ID: document.ID, Md5: fileHash, Filepath: &fileName, - }) - if err != nil { + }); err != nil { + log.Error("[uploadDocumentFile] UpsertDocument DB Error:", err) c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Document Error"}) return } // Update Document Sync Attribute - _, err = api.DB.Queries.UpdateDocumentSync(api.DB.Ctx, database.UpdateDocumentSyncParams{ + if _, err = api.DB.Queries.UpdateDocumentSync(api.DB.Ctx, database.UpdateDocumentSyncParams{ ID: document.ID, Synced: true, - }) - if err != nil { + }); err != nil { + log.Error("[uploadDocumentFile] UpdateDocumentSync DB Error:", err) c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Document"}) return } @@ -558,6 +570,7 @@ func (api *API) uploadDocumentFile(c *gin.Context) { func (api *API) downloadDocumentFile(c *gin.Context) { var rDoc requestDocumentID if err := c.ShouldBindUri(&rDoc); err != nil { + log.Error("[downloadDocumentFile] Invalid URI Bind") c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Request"}) return } @@ -565,11 +578,13 @@ func (api *API) downloadDocumentFile(c *gin.Context) { // Get Document document, err := api.DB.Queries.GetDocument(api.DB.Ctx, rDoc.DocumentID) if err != nil { + log.Error("[uploadDocumentFile] GetDocument DB Error:", err) c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Unknown Document"}) return } if document.Filepath == nil { + log.Error("[uploadDocumentFile] Document Doesn't Have File:", rDoc.DocumentID) c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Document Doesn't Exist"}) return } @@ -580,6 +595,7 @@ func (api *API) downloadDocumentFile(c *gin.Context) { // Validate File Exists _, err = os.Stat(filePath) if os.IsNotExist(err) { + log.Error("[uploadDocumentFile] File Doesn't Exist:", rDoc.DocumentID) c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Document Doesn't Exists"}) return } diff --git a/api/web-routes.go b/api/web-routes.go index 4f7f4e1..8a4dded 100644 --- a/api/web-routes.go +++ b/api/web-routes.go @@ -135,7 +135,7 @@ func (api *API) getActivity(c *gin.Context) { } if activity == nil { - activity = []database.Activity{} + activity = []database.GetActivityRow{} } c.JSON(http.StatusOK, activity) diff --git a/database/models.go b/database/models.go index d641ff4..19eb8a3 100644 --- a/database/models.go +++ b/database/models.go @@ -72,8 +72,9 @@ type RescaledActivity struct { } type User struct { - ID string `json:"id"` - Pass string `json:"-"` - Admin bool `json:"-"` - CreatedAt time.Time `json:"created_at"` + ID string `json:"id"` + Pass string `json:"-"` + Admin bool `json:"-"` + TimeOffset string `json:"time_offset"` + CreatedAt time.Time `json:"created_at"` } diff --git a/database/query.sql b/database/query.sql index 611fd43..3fe1c50 100644 --- a/database/query.sql +++ b/database/query.sql @@ -126,7 +126,7 @@ WHERE SELECT CAST(value AS TEXT) AS id, CAST((documents.filepath IS NULL) AS BOOLEAN) AS want_file, - CAST((documents.synced != true) AS BOOLEAN) AS want_metadata + CAST((IFNULL(documents.synced, false) != true) AS BOOLEAN) AS want_metadata FROM json_each(?1) LEFT JOIN documents ON value = documents.id @@ -134,8 +134,8 @@ WHERE ( documents.id IS NOT NULL AND documents.deleted = false AND ( - documents.synced = false - OR documents.filepath IS NULL + documents.synced = false + OR documents.filepath IS NULL ) ) OR (documents.id IS NULL) @@ -174,10 +174,7 @@ SELECT CAST(IFNULL(current_page, 0) AS INTEGER) AS current_page, CAST(IFNULL(total_pages, 0) AS INTEGER) AS total_pages, CAST(IFNULL(total_time_minutes, 0) AS INTEGER) AS total_time_minutes, - - CAST( - STRFTIME('%Y-%m-%dT%H:%M:%SZ', IFNULL(last_read, "1970-01-01") - ) AS TEXT) AS last_read, + CAST(DATETIME(IFNULL(last_read, "1970-01-01"), time_offset) AS TEXT) AS last_read, CAST(CASE WHEN percentage > 97.0 THEN 100.0 @@ -186,8 +183,9 @@ SELECT END AS REAL) AS percentage FROM documents -LEFT JOIN true_progress ON document_id = id -ORDER BY last_read DESC, created_at DESC +LEFT JOIN true_progress ON true_progress.document_id = documents.id +LEFT JOIN users ON users.id = $user_id +ORDER BY true_progress.last_read DESC, documents.created_at DESC LIMIT $limit OFFSET $offset; @@ -206,13 +204,24 @@ LIMIT $limit OFFSET $offset; -- name: GetActivity :many -SELECT * FROM activity +SELECT + document_id, + CAST(DATETIME(activity.start_time, time_offset) AS TEXT) AS start_time, + title, + author, + duration, + current_page, + total_pages +FROM activity +LEFT JOIN documents ON documents.id = activity.document_id +LEFT JOIN users ON users.id = activity.user_id WHERE - user_id = $user_id + activity.user_id = $user_id AND ( - ($doc_filter = TRUE AND document_id = $document_id) - OR $doc_filter = FALSE + CAST($doc_filter AS BOOLEAN) = TRUE + AND document_id = $document_id ) + OR $doc_filter = FALSE ORDER BY start_time DESC LIMIT $limit OFFSET $offset; @@ -249,8 +258,9 @@ FROM capped_stats; -- name: GetDocumentDaysRead :one WITH document_days AS ( - SELECT DATE(start_time, 'localtime') AS dates + SELECT DATE(start_time, time_offset) AS dates FROM rescaled_activity + JOIN users ON users.id = rescaled_activity.user_id WHERE document_id = $document_id AND user_id = $user_id GROUP BY dates @@ -261,12 +271,11 @@ FROM document_days; -- name: GetUserWindowStreaks :one WITH document_windows AS ( SELECT CASE - -- TODO: Timezones! E.g. DATE(start_time, '-5 hours') - -- TODO: Timezones! E.g. DATE(start_time, '-5 hours', '-7 days') - WHEN ?2 = "WEEK" THEN STRFTIME('%Y-%m-%d', start_time, 'weekday 0', '-7 day') - WHEN ?2 = "DAY" THEN DATE(start_time) + WHEN ?2 = "WEEK" THEN STRFTIME('%Y-%m-%d', start_time, 'weekday 0', '-7 day', time_offset) + WHEN ?2 = "DAY" THEN DATE(start_time, time_offset) END AS read_window FROM activity + JOIN users ON users.id = activity.user_id WHERE user_id = $user_id AND CAST($window AS TEXT) = CAST($window AS TEXT) GROUP BY read_window @@ -287,8 +296,8 @@ streaks AS ( MAX(read_window) AS end_date FROM partitions GROUP BY CASE - WHEN ?2 = "DAY" THEN DATE(read_window, '+' || seqnum || ' day') - WHEN ?2 = "WEEK" THEN DATE(read_window, '+' || (seqnum * 7) || ' day') + WHEN ?2 = "DAY" THEN DATE(read_window, '+' || seqnum || ' day') + WHEN ?2 = "WEEK" THEN DATE(read_window, '+' || (seqnum * 7) || ' day') END ORDER BY end_date DESC ), @@ -331,8 +340,9 @@ SELECT LIMIT 1; -- name: GetDailyReadStats :many -WITH RECURSIVE last_30_days (date) AS ( - SELECT DATE('now') AS date +WITH RECURSIVE last_30_days AS ( + SELECT DATE('now', time_offset) AS date + FROM users WHERE users.id = $user_id UNION ALL SELECT DATE(date, '-1 days') FROM last_30_days @@ -341,8 +351,9 @@ WITH RECURSIVE last_30_days (date) AS ( activity_records AS ( SELECT sum(duration) AS seconds_read, - DATE(start_time, 'localtime') AS day + DATE(start_time, time_offset) AS day FROM activity + LEFT JOIN users ON users.id = activity.user_id WHERE user_id = $user_id GROUP BY day ORDER BY day DESC @@ -358,11 +369,3 @@ FROM last_30_days LEFT JOIN activity_records ON activity_records.day == last_30_days.date ORDER BY date DESC LIMIT 30; - --- SELECT --- sum(duration) / 60 AS minutes_read, --- DATE(start_time, 'localtime') AS day --- FROM activity --- GROUP BY day --- ORDER BY day DESC --- LIMIT 10; diff --git a/database/query.sql.go b/database/query.sql.go index ebef836..ac51901 100644 --- a/database/query.sql.go +++ b/database/query.sql.go @@ -96,27 +96,48 @@ func (q *Queries) DeleteDocument(ctx context.Context, id string) (int64, error) } const getActivity = `-- name: GetActivity :many -SELECT id, user_id, document_id, device_id, start_time, duration, current_page, total_pages, created_at FROM activity +SELECT + document_id, + CAST(DATETIME(activity.start_time, time_offset) AS TEXT) AS start_time, + title, + author, + duration, + current_page, + total_pages +FROM activity +LEFT JOIN documents ON documents.id = activity.document_id +LEFT JOIN users ON users.id = activity.user_id WHERE - user_id = ?1 + activity.user_id = ?1 AND ( - (?2 = TRUE AND document_id = ?3) - OR ?2 = FALSE + CAST(?2 AS BOOLEAN) = TRUE + AND document_id = ?3 ) + OR ?2 = FALSE ORDER BY start_time DESC LIMIT ?5 OFFSET ?4 ` type GetActivityParams struct { - UserID string `json:"user_id"` - DocFilter interface{} `json:"doc_filter"` - DocumentID string `json:"document_id"` - Offset int64 `json:"offset"` - Limit int64 `json:"limit"` + UserID string `json:"user_id"` + DocFilter bool `json:"doc_filter"` + DocumentID string `json:"document_id"` + Offset int64 `json:"offset"` + Limit int64 `json:"limit"` } -func (q *Queries) GetActivity(ctx context.Context, arg GetActivityParams) ([]Activity, error) { +type GetActivityRow struct { + DocumentID string `json:"document_id"` + StartTime string `json:"start_time"` + Title *string `json:"title"` + Author *string `json:"author"` + Duration int64 `json:"duration"` + CurrentPage int64 `json:"current_page"` + TotalPages int64 `json:"total_pages"` +} + +func (q *Queries) GetActivity(ctx context.Context, arg GetActivityParams) ([]GetActivityRow, error) { rows, err := q.db.QueryContext(ctx, getActivity, arg.UserID, arg.DocFilter, @@ -128,19 +149,17 @@ func (q *Queries) GetActivity(ctx context.Context, arg GetActivityParams) ([]Act return nil, err } defer rows.Close() - var items []Activity + var items []GetActivityRow for rows.Next() { - var i Activity + var i GetActivityRow if err := rows.Scan( - &i.ID, - &i.UserID, &i.DocumentID, - &i.DeviceID, &i.StartTime, + &i.Title, + &i.Author, &i.Duration, &i.CurrentPage, &i.TotalPages, - &i.CreatedAt, ); err != nil { return nil, err } @@ -156,8 +175,9 @@ func (q *Queries) GetActivity(ctx context.Context, arg GetActivityParams) ([]Act } const getDailyReadStats = `-- name: GetDailyReadStats :many -WITH RECURSIVE last_30_days (date) AS ( - SELECT DATE('now') AS date +WITH RECURSIVE last_30_days AS ( + SELECT DATE('now', time_offset) AS date + FROM users WHERE users.id = ?1 UNION ALL SELECT DATE(date, '-1 days') FROM last_30_days @@ -166,8 +186,9 @@ WITH RECURSIVE last_30_days (date) AS ( activity_records AS ( SELECT sum(duration) AS seconds_read, - DATE(start_time, 'localtime') AS day + DATE(start_time, time_offset) AS day FROM activity + LEFT JOIN users ON users.id = activity.user_id WHERE user_id = ?1 GROUP BY day ORDER BY day DESC @@ -372,8 +393,9 @@ func (q *Queries) GetDocument(ctx context.Context, documentID string) (Document, const getDocumentDaysRead = `-- name: GetDocumentDaysRead :one WITH document_days AS ( - SELECT DATE(start_time, 'localtime') AS dates + SELECT DATE(start_time, time_offset) AS dates FROM rescaled_activity + JOIN users ON users.id = rescaled_activity.user_id WHERE document_id = ?1 AND user_id = ?2 GROUP BY dates @@ -521,7 +543,7 @@ WITH true_progress AS ( total_pages, ROUND(CAST(current_page AS REAL) / CAST(total_pages AS REAL) * 100, 2) AS percentage FROM activity - WHERE user_id = ?3 + WHERE user_id = ?1 GROUP BY document_id HAVING MAX(start_time) ) @@ -531,10 +553,7 @@ SELECT CAST(IFNULL(current_page, 0) AS INTEGER) AS current_page, CAST(IFNULL(total_pages, 0) AS INTEGER) AS total_pages, CAST(IFNULL(total_time_minutes, 0) AS INTEGER) AS total_time_minutes, - - CAST( - STRFTIME('%Y-%m-%dT%H:%M:%SZ', IFNULL(last_read, "1970-01-01") - ) AS TEXT) AS last_read, + CAST(DATETIME(IFNULL(last_read, "1970-01-01"), time_offset) AS TEXT) AS last_read, CAST(CASE WHEN percentage > 97.0 THEN 100.0 @@ -543,16 +562,17 @@ SELECT END AS REAL) AS percentage FROM documents -LEFT JOIN true_progress ON document_id = id -ORDER BY last_read DESC, created_at DESC -LIMIT ?2 -OFFSET ?1 +LEFT JOIN true_progress ON true_progress.document_id = documents.id +LEFT JOIN users ON users.id = ?1 +ORDER BY true_progress.last_read DESC, documents.created_at DESC +LIMIT ?3 +OFFSET ?2 ` type GetDocumentsWithStatsParams struct { + UserID string `json:"user_id"` Offset int64 `json:"offset"` Limit int64 `json:"limit"` - UserID string `json:"user_id"` } type GetDocumentsWithStatsRow struct { @@ -578,7 +598,7 @@ type GetDocumentsWithStatsRow struct { } func (q *Queries) GetDocumentsWithStats(ctx context.Context, arg GetDocumentsWithStatsParams) ([]GetDocumentsWithStatsRow, error) { - rows, err := q.db.QueryContext(ctx, getDocumentsWithStats, arg.Offset, arg.Limit, arg.UserID) + rows, err := q.db.QueryContext(ctx, getDocumentsWithStats, arg.UserID, arg.Offset, arg.Limit) if err != nil { return nil, err } @@ -742,7 +762,7 @@ func (q *Queries) GetProgress(ctx context.Context, arg GetProgressParams) (GetPr } const getUser = `-- name: GetUser :one -SELECT id, pass, admin, created_at FROM users +SELECT id, pass, admin, time_offset, created_at FROM users WHERE id = ?1 LIMIT 1 ` @@ -753,6 +773,7 @@ func (q *Queries) GetUser(ctx context.Context, userID string) (User, error) { &i.ID, &i.Pass, &i.Admin, + &i.TimeOffset, &i.CreatedAt, ) return i, err @@ -761,12 +782,11 @@ func (q *Queries) GetUser(ctx context.Context, userID string) (User, error) { const getUserWindowStreaks = `-- name: GetUserWindowStreaks :one WITH document_windows AS ( SELECT CASE - -- TODO: Timezones! E.g. DATE(start_time, '-5 hours') - -- TODO: Timezones! E.g. DATE(start_time, '-5 hours', '-7 days') - WHEN ?2 = "WEEK" THEN STRFTIME('%Y-%m-%d', start_time, 'weekday 0', '-7 day') - WHEN ?2 = "DAY" THEN DATE(start_time) + WHEN ?2 = "WEEK" THEN STRFTIME('%Y-%m-%d', start_time, 'weekday 0', '-7 day', time_offset) + WHEN ?2 = "DAY" THEN DATE(start_time, time_offset) END AS read_window FROM activity + JOIN users ON users.id = activity.user_id WHERE user_id = ?1 AND CAST(?2 AS TEXT) = CAST(?2 AS TEXT) GROUP BY read_window @@ -787,8 +807,8 @@ streaks AS ( MAX(read_window) AS end_date FROM partitions GROUP BY CASE - WHEN ?2 = "DAY" THEN DATE(read_window, '+' || seqnum || ' day') - WHEN ?2 = "WEEK" THEN DATE(read_window, '+' || (seqnum * 7) || ' day') + WHEN ?2 = "DAY" THEN DATE(read_window, '+' || seqnum || ' day') + WHEN ?2 = "WEEK" THEN DATE(read_window, '+' || (seqnum * 7) || ' day') END ORDER BY end_date DESC ), @@ -852,7 +872,7 @@ func (q *Queries) GetUserWindowStreaks(ctx context.Context, arg GetUserWindowStr } const getUsers = `-- name: GetUsers :many -SELECT id, pass, admin, created_at FROM users +SELECT id, pass, admin, time_offset, created_at FROM users WHERE users.id = ?1 OR ?1 IN ( @@ -885,6 +905,7 @@ func (q *Queries) GetUsers(ctx context.Context, arg GetUsersParams) ([]User, err &i.ID, &i.Pass, &i.Admin, + &i.TimeOffset, &i.CreatedAt, ); err != nil { return nil, err @@ -904,7 +925,7 @@ const getWantedDocuments = `-- name: GetWantedDocuments :many SELECT CAST(value AS TEXT) AS id, CAST((documents.filepath IS NULL) AS BOOLEAN) AS want_file, - CAST((documents.synced != true) AS BOOLEAN) AS want_metadata + CAST((IFNULL(documents.synced, false) != true) AS BOOLEAN) AS want_metadata FROM json_each(?1) LEFT JOIN documents ON value = documents.id @@ -912,8 +933,8 @@ WHERE ( documents.id IS NOT NULL AND documents.deleted = false AND ( - documents.synced = false - OR documents.filepath IS NULL + documents.synced = false + OR documents.filepath IS NULL ) ) OR (documents.id IS NULL) diff --git a/database/schema.sql b/database/schema.sql index f0e895f..ebff825 100644 --- a/database/schema.sql +++ b/database/schema.sql @@ -6,6 +6,7 @@ CREATE TABLE IF NOT EXISTS users ( pass TEXT NOT NULL, admin BOOLEAN NOT NULL DEFAULT 0 CHECK (admin IN (0, 1)), + time_offset TEXT NOT NULL DEFAULT '0 hours', created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ); diff --git a/templates/activity.html b/templates/activity.html index 0119372..b1dbb3e 100644 --- a/templates/activity.html +++ b/templates/activity.html @@ -1,4 +1,56 @@ {{template "base.html" .}} {{define "title"}}Activity{{end}} {{define "content"}} -

Activity

+ +
+
+ + + + + + + + + + + {{range $activity := .Data }} + + + + + + + {{end}} + +
+ Document + + Time + + Duration + + Page +
+

{{ $activity.Author }} - {{ $activity.Title }}

+ 		+

{{ $activity.StartTime }}

+ 		+

{{ $activity.Duration }}

+ 		+

{{ $activity.CurrentPage }} / {{ $activity.TotalPages }}

+
+
+
{{end}} diff --git a/templates/base.html b/templates/base.html index a6f3c5b..93dcbb4 100644 --- a/templates/base.html +++ b/templates/base.html @@ -11,9 +11,28 @@ class="relative h-screen overflow-hidden bg-gray-100 dark:bg-gray-800" >
-