chore: add various tests

.drone.yml

@@ -1,7 +1,11 @@
 kind: pipeline
-type: kubernetes
+type: docker
 name: default
 
+trigger:
+  branch:
+    - master
+
 steps:
   # Unit Tests
   - name: tests
@@ -23,6 +27,8 @@ steps:
       registry: gitea.va.reichard.io
       tags:
         - dev
+      custom_dns:
+        - 8.8.8.8
       username:
         from_secret: docker_username
       password:

.envrc

@@ -1 +1 @@
-use nix
+use flake

Dockerfile

@@ -3,7 +3,7 @@ FROM alpine AS alpine
 RUN apk update && apk add --no-cache ca-certificates tzdata
 
 # Build Image
-FROM golang:1.21 AS build
+FROM golang:1.24 AS build
 
 # Create Package Directory
 RUN mkdir -p /opt/antholume

Makefile

@@ -27,7 +27,7 @@ docker_build_release_latest: build_tailwind
 		--push .
 
 build_tailwind:
-	tailwind build -o ./assets/style.css --minify
+	tailwindcss build -o ./assets/style.css --minify
 
 dev: build_tailwind
 	GIN_MODE=release \

api/api.go

@@ -325,6 +325,13 @@ func (api *API) loadTemplates(
 	return nil
 }
 
+func (api *API) templateMiddleware(router *gin.Engine) gin.HandlerFunc {
+	return func(c *gin.Context) {
+		router.HTMLRender = *api.generateTemplates()
+		c.Next()
+	}
+}
+
 func loggingMiddleware(c *gin.Context) {
 	// Start timer
 	startTime := time.Now()
@@ -360,10 +367,3 @@ func loggingMiddleware(c *gin.Context) {
 	// Log result
 	log.WithFields(logData).Info(fmt.Sprintf("%s %s", c.Request.Method, c.Request.URL.Path))
 }
-
-func (api *API) templateMiddleware(router *gin.Engine) gin.HandlerFunc {
-	return func(c *gin.Context) {
-		router.HTMLRender = *api.generateTemplates()
-		c.Next()
-	}
-}

api/app-admin-routes.go

@@ -3,6 +3,7 @@ package api
 import (
 	"archive/zip"
 	"bufio"
+	"context"
 	"crypto/md5"
 	"encoding/json"
 	"fmt"
@@ -112,7 +113,7 @@ func (api *API) appPerformAdminAction(c *gin.Context) {
 		// 2. Select all / deselect?
 	case adminCacheTables:
 		go func() {
-			err := api.db.CacheTempTables()
+			err := api.db.CacheTempTables(c)
 			if err != nil {
 				log.Error("Unable to cache temp tables: ", err)
 			}
@@ -122,7 +123,7 @@ func (api *API) appPerformAdminAction(c *gin.Context) {
 		return
 	case adminBackup:
 		// Vacuum
-		_, err := api.db.DB.ExecContext(api.db.Ctx, "VACUUM;")
+		_, err := api.db.DB.ExecContext(c, "VACUUM;")
 		if err != nil {
 			log.Error("Unable to vacuum DB: ", err)
 			appErrorPage(c, http.StatusInternalServerError, "Unable to vacuum database")
@@ -144,7 +145,7 @@ func (api *API) appPerformAdminAction(c *gin.Context) {
 				}
 			}
 
-			err := api.createBackup(w, directories)
+			err := api.createBackup(c, w, directories)
 			if err != nil {
 				log.Error("Backup Error: ", err)
 			}
@@ -174,7 +175,10 @@ func (api *API) appGetAdminLogs(c *gin.Context) {
 	rAdminLogs.Filter = strings.TrimSpace(rAdminLogs.Filter)
 
 	var jqFilter *gojq.Code
-	if rAdminLogs.Filter != "" {
+	var basicFilter string
+	if strings.HasPrefix(rAdminLogs.Filter, "\"") && strings.HasSuffix(rAdminLogs.Filter, "\"") {
+		basicFilter = rAdminLogs.Filter[1 : len(rAdminLogs.Filter)-1]
+	} else if rAdminLogs.Filter != "" {
 		parsed, err := gojq.Parse(rAdminLogs.Filter)
 		if err != nil {
 			log.Error("Unable to parse JQ filter")
@@ -220,12 +224,17 @@ func (api *API) appGetAdminLogs(c *gin.Context) {
 			continue
 		}
 
-		// No Filter
-		if jqFilter == nil {
+		// Basic Filter
+		if basicFilter != "" && strings.Contains(string(rawData), basicFilter) {
 			logLines = append(logLines, string(rawData))
 			continue
 		}
 
+		// No JQ Filter
+		if jqFilter == nil {
+			continue
+		}
+
 		// Error or nil
 		result, _ := jqFilter.Run(jsonMap).Next()
 		if _, ok := result.(error); ok {
@@ -253,7 +262,7 @@ func (api *API) appGetAdminLogs(c *gin.Context) {
 func (api *API) appGetAdminUsers(c *gin.Context) {
 	templateVars, _ := api.getBaseTemplateVars("admin-users", c)
 
-	users, err := api.db.Queries.GetUsers(api.db.Ctx)
+	users, err := api.db.Queries.GetUsers(c)
 	if err != nil {
 		log.Error("GetUsers DB Error: ", err)
 		appErrorPage(c, http.StatusInternalServerError, fmt.Sprintf("GetUsers DB Error: %v", err))
@@ -284,11 +293,11 @@ func (api *API) appUpdateAdminUsers(c *gin.Context) {
 	var err error
 	switch rUpdate.Operation {
 	case opCreate:
-		err = api.createUser(rUpdate.User, rUpdate.Password, rUpdate.IsAdmin)
+		err = api.createUser(c, rUpdate.User, rUpdate.Password, rUpdate.IsAdmin)
 	case opUpdate:
-		err = api.updateUser(rUpdate.User, rUpdate.Password, rUpdate.IsAdmin)
+		err = api.updateUser(c, rUpdate.User, rUpdate.Password, rUpdate.IsAdmin)
 	case opDelete:
-		err = api.deleteUser(rUpdate.User)
+		err = api.deleteUser(c, rUpdate.User)
 	default:
 		appErrorPage(c, http.StatusNotFound, "Unknown user operation")
 		return
@@ -299,7 +308,7 @@ func (api *API) appUpdateAdminUsers(c *gin.Context) {
 		return
 	}
 
-	users, err := api.db.Queries.GetUsers(api.db.Ctx)
+	users, err := api.db.Queries.GetUsers(c)
 	if err != nil {
 		log.Error("GetUsers DB Error: ", err)
 		appErrorPage(c, http.StatusInternalServerError, fmt.Sprintf("GetUsers DB Error: %v", err))
@@ -440,7 +449,7 @@ func (api *API) appPerformAdminImport(c *gin.Context) {
 		iResult.Name = fmt.Sprintf("%s - %s", *fileMeta.Author, *fileMeta.Title)
 
 		// Check already exists
-		_, err = qtx.GetDocument(api.db.Ctx, *fileMeta.PartialMD5)
+		_, err = qtx.GetDocument(c, *fileMeta.PartialMD5)
 		if err == nil {
 			log.Warnf("document already exists: %s", *fileMeta.PartialMD5)
 			iResult.Status = importExists
@@ -484,7 +493,7 @@ func (api *API) appPerformAdminImport(c *gin.Context) {
 		}
 
 		// Upsert document
-		if _, err = qtx.UpsertDocument(api.db.Ctx, database.UpsertDocumentParams{
+		if _, err = qtx.UpsertDocument(c, database.UpsertDocumentParams{
 			ID:          *fileMeta.PartialMD5,
 			Title:       fileMeta.Title,
 			Author:      fileMeta.Author,
@@ -619,7 +628,7 @@ func (api *API) processRestoreFile(rAdminAction requestAdminAction, c *gin.Conte
 
 	// Save Backup File
 	w := bufio.NewWriter(backupFile)
-	err = api.createBackup(w, []string{"covers", "documents"})
+	err = api.createBackup(c, w, []string{"covers", "documents"})
 	if err != nil {
 		log.Error("Unable to save backup file: ", err)
 		appErrorPage(c, http.StatusInternalServerError, "Unable to save backup file")
@@ -642,13 +651,13 @@ func (api *API) processRestoreFile(rAdminAction requestAdminAction, c *gin.Conte
 	}
 
 	// Reinit DB
-	if err := api.db.Reload(); err != nil {
+	if err := api.db.Reload(c); err != nil {
 		appErrorPage(c, http.StatusInternalServerError, "Unable to reload DB")
 		log.Panicf("Unable to reload DB: %v", err)
 	}
 
 	// Rotate Auth Hashes
-	if err := api.rotateAllAuthHashes(); err != nil {
+	if err := api.rotateAllAuthHashes(c); err != nil {
 		appErrorPage(c, http.StatusInternalServerError, "Unable to rotate hashes")
 		log.Panicf("Unable to rotate auth hashes: %v", err)
 	}
@@ -709,9 +718,9 @@ func (api *API) removeData() error {
 	return nil
 }
 
-func (api *API) createBackup(w io.Writer, directories []string) error {
+func (api *API) createBackup(ctx context.Context, w io.Writer, directories []string) error {
 	// Vacuum DB
-	_, err := api.db.DB.ExecContext(api.db.Ctx, "VACUUM;")
+	_, err := api.db.DB.ExecContext(ctx, "VACUUM;")
 	if err != nil {
 		return errors.Wrap(err, "Unable to vacuum database")
 	}
@@ -784,8 +793,8 @@ func (api *API) createBackup(w io.Writer, directories []string) error {
 	return nil
 }
 
-func (api *API) isLastAdmin(userID string) (bool, error) {
-	allUsers, err := api.db.Queries.GetUsers(api.db.Ctx)
+func (api *API) isLastAdmin(ctx context.Context, userID string) (bool, error) {
+	allUsers, err := api.db.Queries.GetUsers(ctx)
 	if err != nil {
 		return false, errors.Wrap(err, fmt.Sprintf("GetUsers DB Error: %v", err))
 	}
@@ -801,7 +810,7 @@ func (api *API) isLastAdmin(userID string) (bool, error) {
 	return !hasAdmin, nil
 }
 
-func (api *API) createUser(user string, rawPassword *string, isAdmin *bool) error {
+func (api *API) createUser(ctx context.Context, user string, rawPassword *string, isAdmin *bool) error {
 	// Validate Necessary Parameters
 	if rawPassword == nil || *rawPassword == "" {
 		return fmt.Errorf("password can't be empty")
@@ -836,7 +845,7 @@ func (api *API) createUser(user string, rawPassword *string, isAdmin *bool) erro
 	createParams.AuthHash = &authHash
 
 	// Create user in DB
-	if rows, err := api.db.Queries.CreateUser(api.db.Ctx, createParams); err != nil {
+	if rows, err := api.db.Queries.CreateUser(ctx, createParams); err != nil {
 		log.Error("CreateUser DB Error:", err)
 		return fmt.Errorf("unable to create user")
 	} else if rows == 0 {
@@ -847,7 +856,7 @@ func (api *API) createUser(user string, rawPassword *string, isAdmin *bool) erro
 	return nil
 }
 
-func (api *API) updateUser(user string, rawPassword *string, isAdmin *bool) error {
+func (api *API) updateUser(ctx context.Context, user string, rawPassword *string, isAdmin *bool) error {
 	// Validate Necessary Parameters
 	if rawPassword == nil && isAdmin == nil {
 		return fmt.Errorf("nothing to update")
@@ -862,7 +871,7 @@ func (api *API) updateUser(user string, rawPassword *string, isAdmin *bool) erro
 	if isAdmin != nil {
 		updateParams.Admin = *isAdmin
 	} else {
-		user, err := api.db.Queries.GetUser(api.db.Ctx, user)
+		user, err := api.db.Queries.GetUser(ctx, user)
 		if err != nil {
 			return errors.Wrap(err, fmt.Sprintf("GetUser DB Error: %v", err))
 		}
@@ -870,7 +879,7 @@ func (api *API) updateUser(user string, rawPassword *string, isAdmin *bool) erro
 	}
 
 	// Check Admins - Disallow Demotion
-	if isLast, err := api.isLastAdmin(user); err != nil {
+	if isLast, err := api.isLastAdmin(ctx, user); err != nil {
 		return err
 	} else if isLast && !updateParams.Admin {
 		return fmt.Errorf("unable to demote %s - last admin", user)
@@ -900,7 +909,7 @@ func (api *API) updateUser(user string, rawPassword *string, isAdmin *bool) erro
 	}
 
 	// Update User
-	_, err := api.db.Queries.UpdateUser(api.db.Ctx, updateParams)
+	_, err := api.db.Queries.UpdateUser(ctx, updateParams)
 	if err != nil {
 		return errors.Wrap(err, fmt.Sprintf("UpdateUser DB Error: %v", err))
 	}
@@ -908,9 +917,9 @@ func (api *API) updateUser(user string, rawPassword *string, isAdmin *bool) erro
 	return nil
 }
 
-func (api *API) deleteUser(user string) error {
+func (api *API) deleteUser(ctx context.Context, user string) error {
 	// Check Admins
-	if isLast, err := api.isLastAdmin(user); err != nil {
+	if isLast, err := api.isLastAdmin(ctx, user); err != nil {
 		return err
 	} else if isLast {
 		return fmt.Errorf("unable to delete %s - last admin", user)
@@ -926,13 +935,13 @@ func (api *API) deleteUser(user string) error {
 
 	// Save Backup File (DB Only)
 	w := bufio.NewWriter(backupFile)
-	err = api.createBackup(w, []string{})
+	err = api.createBackup(ctx, w, []string{})
 	if err != nil {
 		return err
 	}
 
 	// Delete User
-	_, err = api.db.Queries.DeleteUser(api.db.Ctx, user)
+	_, err = api.db.Queries.DeleteUser(ctx, user)
 	if err != nil {
 		return errors.Wrap(err, fmt.Sprintf("DeleteUser DB Error: %v", err))
 	}

api/app-routes.go

@@ -1,6 +1,7 @@
 package api
 
 import (
+	"context"
 	"crypto/md5"
 	"database/sql"
 	"fmt"
@@ -22,8 +23,8 @@ import (
 	"golang.org/x/exp/slices"
 	"reichard.io/antholume/database"
 	"reichard.io/antholume/metadata"
+	"reichard.io/antholume/pkg/ptr"
 	"reichard.io/antholume/search"
-	"reichard.io/antholume/utils"
 )
 
 type backupType string
@@ -110,11 +111,12 @@ func (api *API) appGetDocuments(c *gin.Context) {
 		query = &search
 	}
 
-	documents, err := api.db.Queries.GetDocumentsWithStats(api.db.Ctx, database.GetDocumentsWithStatsParams{
-		UserID: auth.UserName,
-		Query:  query,
-		Offset: (*qParams.Page - 1) * *qParams.Limit,
-		Limit:  *qParams.Limit,
+	documents, err := api.db.Queries.GetDocumentsWithStats(c, database.GetDocumentsWithStatsParams{
+		UserID:  auth.UserName,
+		Query:   query,
+		Deleted: ptr.Of(false),
+		Offset:  (*qParams.Page - 1) * *qParams.Limit,
+		Limit:   *qParams.Limit,
 	})
 	if err != nil {
 		log.Error("GetDocumentsWithStats DB Error: ", err)
@@ -122,14 +124,14 @@ func (api *API) appGetDocuments(c *gin.Context) {
 		return
 	}
 
-	length, err := api.db.Queries.GetDocumentsSize(api.db.Ctx, query)
+	length, err := api.db.Queries.GetDocumentsSize(c, query)
 	if err != nil {
 		log.Error("GetDocumentsSize DB Error: ", err)
 		appErrorPage(c, http.StatusInternalServerError, fmt.Sprintf("GetDocumentsSize DB Error: %v", err))
 		return
 	}
 
-	if err = api.getDocumentsWordCount(documents); err != nil {
+	if err = api.getDocumentsWordCount(c, documents); err != nil {
 		log.Error("Unable to Get Word Counts: ", err)
 	}
 
@@ -161,13 +163,10 @@ func (api *API) appGetDocument(c *gin.Context) {
 		return
 	}
 
-	document, err := api.db.Queries.GetDocumentWithStats(api.db.Ctx, database.GetDocumentWithStatsParams{
-		UserID:     auth.UserName,
-		DocumentID: rDocID.DocumentID,
-	})
+	document, err := api.db.GetDocument(c, rDocID.DocumentID, auth.UserName)
 	if err != nil {
-		log.Error("GetDocumentWithStats DB Error: ", err)
-		appErrorPage(c, http.StatusInternalServerError, fmt.Sprintf("GetDocumentsWithStats DB Error: %v", err))
+		log.Error("GetDocument DB Error: ", err)
+		appErrorPage(c, http.StatusInternalServerError, fmt.Sprintf("GetDocument DB Error: %v", err))
 		return
 	}
 
@@ -193,7 +192,7 @@ func (api *API) appGetProgress(c *gin.Context) {
 		progressFilter.DocumentID = *qParams.Document
 	}
 
-	progress, err := api.db.Queries.GetProgress(api.db.Ctx, progressFilter)
+	progress, err := api.db.Queries.GetProgress(c, progressFilter)
 	if err != nil {
 		log.Error("GetProgress DB Error: ", err)
 		appErrorPage(c, http.StatusInternalServerError, fmt.Sprintf("GetActivity DB Error: %v", err))
@@ -220,7 +219,7 @@ func (api *API) appGetActivity(c *gin.Context) {
 		activityFilter.DocumentID = *qParams.Document
 	}
 
-	activity, err := api.db.Queries.GetActivity(api.db.Ctx, activityFilter)
+	activity, err := api.db.Queries.GetActivity(c, activityFilter)
 	if err != nil {
 		log.Error("GetActivity DB Error: ", err)
 		appErrorPage(c, http.StatusInternalServerError, fmt.Sprintf("GetActivity DB Error: %v", err))
@@ -236,7 +235,7 @@ func (api *API) appGetHome(c *gin.Context) {
 	templateVars, auth := api.getBaseTemplateVars("home", c)
 
 	start := time.Now()
-	graphData, err := api.db.Queries.GetDailyReadStats(api.db.Ctx, auth.UserName)
+	graphData, err := api.db.Queries.GetDailyReadStats(c, auth.UserName)
 	if err != nil {
 		log.Error("GetDailyReadStats DB Error: ", err)
 		appErrorPage(c, http.StatusInternalServerError, fmt.Sprintf("GetDailyReadStats DB Error: %v", err))
@@ -245,7 +244,7 @@ func (api *API) appGetHome(c *gin.Context) {
 	log.Debug("GetDailyReadStats DB Performance: ", time.Since(start))
 
 	start = time.Now()
-	databaseInfo, err := api.db.Queries.GetDatabaseInfo(api.db.Ctx, auth.UserName)
+	databaseInfo, err := api.db.Queries.GetDatabaseInfo(c, auth.UserName)
 	if err != nil {
 		log.Error("GetDatabaseInfo DB Error: ", err)
 		appErrorPage(c, http.StatusInternalServerError, fmt.Sprintf("GetDatabaseInfo DB Error: %v", err))
@@ -254,7 +253,7 @@ func (api *API) appGetHome(c *gin.Context) {
 	log.Debug("GetDatabaseInfo DB Performance: ", time.Since(start))
 
 	start = time.Now()
-	streaks, err := api.db.Queries.GetUserStreaks(api.db.Ctx, auth.UserName)
+	streaks, err := api.db.Queries.GetUserStreaks(c, auth.UserName)
 	if err != nil {
 		log.Error("GetUserStreaks DB Error: ", err)
 		appErrorPage(c, http.StatusInternalServerError, fmt.Sprintf("GetUserStreaks DB Error: %v", err))
@@ -263,7 +262,7 @@ func (api *API) appGetHome(c *gin.Context) {
 	log.Debug("GetUserStreaks DB Performance: ", time.Since(start))
 
 	start = time.Now()
-	userStatistics, err := api.db.Queries.GetUserStatistics(api.db.Ctx)
+	userStatistics, err := api.db.Queries.GetUserStatistics(c)
 	if err != nil {
 		log.Error("GetUserStatistics DB Error: ", err)
 		appErrorPage(c, http.StatusInternalServerError, fmt.Sprintf("GetUserStatistics DB Error: %v", err))
@@ -284,14 +283,14 @@ func (api *API) appGetHome(c *gin.Context) {
 func (api *API) appGetSettings(c *gin.Context) {
 	templateVars, auth := api.getBaseTemplateVars("settings", c)
 
-	user, err := api.db.Queries.GetUser(api.db.Ctx, auth.UserName)
+	user, err := api.db.Queries.GetUser(c, auth.UserName)
 	if err != nil {
 		log.Error("GetUser DB Error: ", err)
 		appErrorPage(c, http.StatusInternalServerError, fmt.Sprintf("GetUser DB Error: %v", err))
 		return
 	}
 
-	devices, err := api.db.Queries.GetDevices(api.db.Ctx, auth.UserName)
+	devices, err := api.db.Queries.GetDevices(c, auth.UserName)
 	if err != nil {
 		log.Error("GetDevices DB Error: ", err)
 		appErrorPage(c, http.StatusInternalServerError, fmt.Sprintf("GetDevices DB Error: %v", err))
@@ -369,7 +368,7 @@ func (api *API) appGetDocumentProgress(c *gin.Context) {
 		return
 	}
 
-	progress, err := api.db.Queries.GetDocumentProgress(api.db.Ctx, database.GetDocumentProgressParams{
+	progress, err := api.db.Queries.GetDocumentProgress(c, database.GetDocumentProgressParams{
 		DocumentID: rDoc.DocumentID,
 		UserID:     auth.UserName,
 	})
@@ -379,13 +378,10 @@ func (api *API) appGetDocumentProgress(c *gin.Context) {
 		return
 	}
 
-	document, err := api.db.Queries.GetDocumentWithStats(api.db.Ctx, database.GetDocumentWithStatsParams{
-		UserID:     auth.UserName,
-		DocumentID: rDoc.DocumentID,
-	})
+	document, err := api.db.GetDocument(c, rDoc.DocumentID, auth.UserName)
 	if err != nil {
-		log.Error("GetDocumentWithStats DB Error: ", err)
-		appErrorPage(c, http.StatusInternalServerError, fmt.Sprintf("GetDocumentWithStats DB Error: %v", err))
+		log.Error("GetDocument DB Error: ", err)
+		appErrorPage(c, http.StatusInternalServerError, fmt.Sprintf("GetDocument DB Error: %v", err))
 		return
 	}
 
@@ -405,7 +401,7 @@ func (api *API) appGetDevices(c *gin.Context) {
 		auth = data.(authData)
 	}
 
-	devices, err := api.db.Queries.GetDevices(api.db.Ctx, auth.UserName)
+	devices, err := api.db.Queries.GetDevices(c, auth.UserName)
 
 	if err != nil && err != sql.ErrNoRows {
 		log.Error("GetDevices DB Error: ", err)
@@ -456,7 +452,7 @@ func (api *API) appUploadNewDocument(c *gin.Context) {
 	}
 
 	// Check Already Exists
-	_, err = api.db.Queries.GetDocument(api.db.Ctx, *metadataInfo.PartialMD5)
+	_, err = api.db.Queries.GetDocument(c, *metadataInfo.PartialMD5)
 	if err == nil {
 		log.Warnf("document already exists: %s", *metadataInfo.PartialMD5)
 		c.Redirect(http.StatusFound, fmt.Sprintf("./documents/%s", *metadataInfo.PartialMD5))
@@ -484,7 +480,7 @@ func (api *API) appUploadNewDocument(c *gin.Context) {
 	}
 
 	// Upsert Document
-	if _, err = api.db.Queries.UpsertDocument(api.db.Ctx, database.UpsertDocumentParams{
+	if _, err = api.db.Queries.UpsertDocument(c, database.UpsertDocumentParams{
 		ID:          *metadataInfo.PartialMD5,
 		Title:       metadataInfo.Title,
 		Author:      metadataInfo.Author,
@@ -574,7 +570,7 @@ func (api *API) appEditDocument(c *gin.Context) {
 
 		coverFileName = &fileName
 	} else if rDocEdit.CoverGBID != nil {
-		var coverDir string = filepath.Join(api.cfg.DataPath, "covers")
+		coverDir := filepath.Join(api.cfg.DataPath, "covers")
 		fileName, err := metadata.CacheCover(*rDocEdit.CoverGBID, coverDir, rDocID.DocumentID, true)
 		if err == nil {
 			coverFileName = fileName
@@ -582,7 +578,7 @@ func (api *API) appEditDocument(c *gin.Context) {
 	}
 
 	// Update Document
-	if _, err := api.db.Queries.UpsertDocument(api.db.Ctx, database.UpsertDocumentParams{
+	if _, err := api.db.Queries.UpsertDocument(c, database.UpsertDocumentParams{
 		ID:          rDocID.DocumentID,
 		Title:       api.sanitizeInput(rDocEdit.Title),
 		Author:      api.sanitizeInput(rDocEdit.Author),
@@ -606,7 +602,7 @@ func (api *API) appDeleteDocument(c *gin.Context) {
 		appErrorPage(c, http.StatusNotFound, "Invalid document")
 		return
 	}
-	changed, err := api.db.Queries.DeleteDocument(api.db.Ctx, rDocID.DocumentID)
+	changed, err := api.db.Queries.DeleteDocument(c, rDocID.DocumentID)
 	if err != nil {
 		log.Error("DeleteDocument DB Error")
 		appErrorPage(c, http.StatusInternalServerError, fmt.Sprintf("DeleteDocument DB Error: %v", err))
@@ -668,7 +664,7 @@ func (api *API) appIdentifyDocument(c *gin.Context) {
 		firstResult := metadataResults[0]
 
 		// Store First Metadata Result
-		if _, err = api.db.Queries.AddMetadata(api.db.Ctx, database.AddMetadataParams{
+		if _, err = api.db.Queries.AddMetadata(c, database.AddMetadataParams{
 			DocumentID:  rDocID.DocumentID,
 			Title:       firstResult.Title,
 			Author:      firstResult.Author,
@@ -687,13 +683,10 @@ func (api *API) appIdentifyDocument(c *gin.Context) {
 		templateVars["MetadataError"] = "No Metadata Found"
 	}
 
-	document, err := api.db.Queries.GetDocumentWithStats(api.db.Ctx, database.GetDocumentWithStatsParams{
-		UserID:     auth.UserName,
-		DocumentID: rDocID.DocumentID,
-	})
+	document, err := api.db.GetDocument(c, rDocID.DocumentID, auth.UserName)
 	if err != nil {
-		log.Error("GetDocumentWithStats DB Error: ", err)
-		appErrorPage(c, http.StatusInternalServerError, fmt.Sprintf("GetDocumentWithStats DB Error: %v", err))
+		log.Error("GetDocument DB Error: ", err)
+		appErrorPage(c, http.StatusInternalServerError, fmt.Sprintf("GetDocument DB Error: %v", err))
 		return
 	}
 
@@ -740,57 +733,50 @@ func (api *API) appSaveNewDocument(c *gin.Context) {
 	}
 
 	// Send Message
-	sendDownloadMessage("Downloading document...", gin.H{"Progress": 10})
+	sendDownloadMessage("Downloading document...", gin.H{"Progress": 1})
+
+	// Scaled Download Function
+	lastTime := time.Now()
+	downloadFunc := func(p float32) {
+		nowTime := time.Now()
+		if nowTime.Before(lastTime.Add(time.Millisecond * 500)) {
+			return
+		}
+		scaledProgress := int((p * 95 / 100) + 2)
+		sendDownloadMessage("Downloading document...", gin.H{"Progress": scaledProgress})
+		lastTime = nowTime
+	}
 
 	// Save Book
-	tempFilePath, err := search.SaveBook(rDocAdd.ID, rDocAdd.Source)
+	tempFilePath, metadata, err := search.SaveBook(rDocAdd.ID, rDocAdd.Source, downloadFunc)
 	if err != nil {
-		log.Warn("Temp File Error: ", err)
+		log.Warn("Save Book Error: ", err)
 		sendDownloadMessage("Unable to download file", gin.H{"Error": true})
 		return
 	}
 
 	// Send Message
-	sendDownloadMessage("Calculating partial MD5...", gin.H{"Progress": 60})
+	sendDownloadMessage("Saving document...", gin.H{"Progress": 98})
 
-	// Calculate Partial MD5 ID
-	partialMD5, err := utils.CalculatePartialMD5(tempFilePath)
-	if err != nil {
-		log.Warn("Partial MD5 Error: ", err)
-		sendDownloadMessage("Unable to calculate partial MD5", gin.H{"Error": true})
+	// Derive Author / Title
+	docAuthor := "Unknown"
+	if *metadata.Author != "" {
+		docAuthor = *metadata.Author
+	} else if *rDocAdd.Author != "" {
+		docAuthor = *rDocAdd.Author
 	}
 
-	// Send Message
-	sendDownloadMessage("Saving file...", gin.H{"Progress": 60})
-
-	// Derive Extension on MIME
-	fileMime, err := mimetype.DetectFile(tempFilePath)
-	if err != nil {
-		log.Warn("MIME Detect Error: ", err)
-		sendDownloadMessage("Unable to download file", gin.H{"Error": true})
-		return
-	}
-	fileExtension := fileMime.Extension()
-
-	// Derive Filename
-	var fileName string
-	if *rDocAdd.Author != "" {
-		fileName = fileName + *rDocAdd.Author
-	} else {
-		fileName = fileName + "Unknown"
+	docTitle := "Unknown"
+	if *metadata.Title != "" {
+		docTitle = *metadata.Title
+	} else if *rDocAdd.Title != "" {
+		docTitle = *rDocAdd.Title
 	}
 
-	if *rDocAdd.Title != "" {
-		fileName = fileName + " - " + *rDocAdd.Title
-	} else {
-		fileName = fileName + " - Unknown"
-	}
-
-	// Remove Slashes
+	// Remove Slashes & Sanitize File Name
+	fileName := fmt.Sprintf("%s - %s", docAuthor, docTitle)
 	fileName = strings.ReplaceAll(fileName, "/", "")
-
-	// Derive & Sanitize File Name
-	fileName = "." + filepath.Clean(fmt.Sprintf("/%s [%s]%s", fileName, *partialMD5, fileExtension))
+	fileName = "." + filepath.Clean(fmt.Sprintf("/%s [%s]%s", fileName, *metadata.PartialMD5, metadata.Type))
 
 	// Open Source File
 	sourceFile, err := os.Open(tempFilePath)
@@ -822,37 +808,15 @@ func (api *API) appSaveNewDocument(c *gin.Context) {
 	}
 
 	// Send Message
-	sendDownloadMessage("Calculating MD5...", gin.H{"Progress": 70})
-
-	// Get MD5 Hash
-	fileHash, err := getFileMD5(safePath)
-	if err != nil {
-		log.Error("Hash Failure: ", err)
-		sendDownloadMessage("Unable to calculate MD5", gin.H{"Error": true})
-		return
-	}
-
-	// Send Message
-	sendDownloadMessage("Calculating word count...", gin.H{"Progress": 80})
-
-	// Get Word Count
-	wordCount, err := metadata.GetWordCount(safePath)
-	if err != nil {
-		log.Error("Word Count Failure: ", err)
-		sendDownloadMessage("Unable to calculate word count", gin.H{"Error": true})
-		return
-	}
-
-	// Send Message
-	sendDownloadMessage("Saving to database...", gin.H{"Progress": 90})
+	sendDownloadMessage("Saving to database...", gin.H{"Progress": 99})
 
 	// Upsert Document
-	if _, err = api.db.Queries.UpsertDocument(api.db.Ctx, database.UpsertDocumentParams{
-		ID:       *partialMD5,
-		Title:    rDocAdd.Title,
-		Author:   rDocAdd.Author,
-		Md5:      fileHash,
-		Words:    wordCount,
+	if _, err = api.db.Queries.UpsertDocument(c, database.UpsertDocumentParams{
+		ID:       *metadata.PartialMD5,
+		Title:    &docTitle,
+		Author:   &docAuthor,
+		Md5:      metadata.MD5,
+		Words:    metadata.WordCount,
 		Filepath: &fileName,
 		Basepath: &basePath,
 	}); err != nil {
@@ -865,7 +829,7 @@ func (api *API) appSaveNewDocument(c *gin.Context) {
 	sendDownloadMessage("Download Success", gin.H{
 		"Progress":   100,
 		"ButtonText": "Go to Book",
-		"ButtonHref": fmt.Sprintf("./documents/%s", *partialMD5),
+		"ButtonHref": fmt.Sprintf("./documents/%s", *metadata.PartialMD5),
 	})
 }
 
@@ -894,7 +858,7 @@ func (api *API) appEditSettings(c *gin.Context) {
 	// Set New Password
 	if rUserSettings.Password != nil && rUserSettings.NewPassword != nil {
 		password := fmt.Sprintf("%x", md5.Sum([]byte(*rUserSettings.Password)))
-		data := api.authorizeCredentials(auth.UserName, password)
+		data := api.authorizeCredentials(c, auth.UserName, password)
 		if data == nil {
 			templateVars["PasswordErrorMessage"] = "Invalid Password"
 		} else {
@@ -916,7 +880,7 @@ func (api *API) appEditSettings(c *gin.Context) {
 	}
 
 	// Update User
-	_, err := api.db.Queries.UpdateUser(api.db.Ctx, newUserSettings)
+	_, err := api.db.Queries.UpdateUser(c, newUserSettings)
 	if err != nil {
 		log.Error("UpdateUser DB Error: ", err)
 		appErrorPage(c, http.StatusInternalServerError, fmt.Sprintf("UpdateUser DB Error: %v", err))
@@ -924,7 +888,7 @@ func (api *API) appEditSettings(c *gin.Context) {
 	}
 
 	// Get User
-	user, err := api.db.Queries.GetUser(api.db.Ctx, auth.UserName)
+	user, err := api.db.Queries.GetUser(c, auth.UserName)
 	if err != nil {
 		log.Error("GetUser DB Error: ", err)
 		appErrorPage(c, http.StatusInternalServerError, fmt.Sprintf("GetUser DB Error: %v", err))
@@ -932,7 +896,7 @@ func (api *API) appEditSettings(c *gin.Context) {
 	}
 
 	// Get Devices
-	devices, err := api.db.Queries.GetDevices(api.db.Ctx, auth.UserName)
+	devices, err := api.db.Queries.GetDevices(c, auth.UserName)
 	if err != nil {
 		log.Error("GetDevices DB Error: ", err)
 		appErrorPage(c, http.StatusInternalServerError, fmt.Sprintf("GetDevices DB Error: %v", err))
@@ -951,7 +915,7 @@ func (api *API) appDemoModeError(c *gin.Context) {
 	appErrorPage(c, http.StatusUnauthorized, "Not Allowed in Demo Mode")
 }
 
-func (api *API) getDocumentsWordCount(documents []database.GetDocumentsWithStatsRow) error {
+func (api *API) getDocumentsWordCount(ctx context.Context, documents []database.GetDocumentsWithStatsRow) error {
 	// Do Transaction
 	tx, err := api.db.DB.Begin()
 	if err != nil {
@@ -974,7 +938,7 @@ func (api *API) getDocumentsWordCount(documents []database.GetDocumentsWithStats
 			if err != nil {
 				log.Warn("Word Count Error: ", err)
 			} else {
-				if _, err := qtx.UpsertDocument(api.db.Ctx, database.UpsertDocumentParams{
+				if _, err := qtx.UpsertDocument(ctx, database.UpsertDocumentParams{
 					ID:    item.ID,
 					Words: wordCount,
 				}); err != nil {
@@ -1035,7 +999,7 @@ func bindQueryParams(c *gin.Context, defaultLimit int64) queryParams {
 }
 
 func appErrorPage(c *gin.Context, errorCode int, errorMessage string) {
-	var errorHuman string = "We're not even sure what happened."
+	errorHuman := "We're not even sure what happened."
 
 	switch errorCode {
 	case http.StatusInternalServerError:

api/auth.go

@@ -1,6 +1,7 @@
 package api
 
 import (
+	"context"
 	"crypto/md5"
 	"fmt"
 	"net/http"
@@ -28,8 +29,8 @@ type authKOHeader struct {
 	AuthKey  string `header:"x-auth-key"`
 }
 
-func (api *API) authorizeCredentials(username string, password string) (auth *authData) {
-	user, err := api.db.Queries.GetUser(api.db.Ctx, username)
+func (api *API) authorizeCredentials(ctx context.Context, username string, password string) (auth *authData) {
+	user, err := api.db.Queries.GetUser(ctx, username)
 	if err != nil {
 		return
 	}
@@ -52,7 +53,7 @@ func (api *API) authKOMiddleware(c *gin.Context) {
 	session := sessions.Default(c)
 
 	// Check Session First
-	if auth, ok := api.getSession(session); ok {
+	if auth, ok := api.getSession(c, session); ok {
 		c.Set("Authorization", auth)
 		c.Header("Cache-Control", "private")
 		c.Next()
@@ -71,7 +72,7 @@ func (api *API) authKOMiddleware(c *gin.Context) {
 		return
 	}
 
-	authData := api.authorizeCredentials(rHeader.AuthUser, rHeader.AuthKey)
+	authData := api.authorizeCredentials(c, rHeader.AuthUser, rHeader.AuthKey)
 	if authData == nil {
 		c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "Unauthorized"})
 		return
@@ -100,7 +101,7 @@ func (api *API) authOPDSMiddleware(c *gin.Context) {
 
 	// Validate Auth
 	password := fmt.Sprintf("%x", md5.Sum([]byte(rawPassword)))
-	authData := api.authorizeCredentials(user, password)
+	authData := api.authorizeCredentials(c, user, password)
 	if authData == nil {
 		c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "Unauthorized"})
 		return
@@ -115,7 +116,7 @@ func (api *API) authWebAppMiddleware(c *gin.Context) {
 	session := sessions.Default(c)
 
 	// Check Session
-	if auth, ok := api.getSession(session); ok {
+	if auth, ok := api.getSession(c, session); ok {
 		c.Set("Authorization", auth)
 		c.Header("Cache-Control", "private")
 		c.Next()
@@ -153,7 +154,7 @@ func (api *API) appAuthLogin(c *gin.Context) {
 
 	// MD5 - KOSync Compatiblity
 	password := fmt.Sprintf("%x", md5.Sum([]byte(rawPassword)))
-	authData := api.authorizeCredentials(username, password)
+	authData := api.authorizeCredentials(c, username, password)
 	if authData == nil {
 		templateVars["Error"] = "Invalid Credentials"
 		c.HTML(http.StatusUnauthorized, "page/login", templateVars)
@@ -208,7 +209,7 @@ func (api *API) appAuthRegister(c *gin.Context) {
 	}
 
 	// Get current users
-	currentUsers, err := api.db.Queries.GetUsers(api.db.Ctx)
+	currentUsers, err := api.db.Queries.GetUsers(c)
 	if err != nil {
 		log.Error("Failed to check all users: ", err)
 		templateVars["Error"] = "Failed to Create User"
@@ -224,7 +225,7 @@ func (api *API) appAuthRegister(c *gin.Context) {
 
 	// Create user in DB
 	authHash := fmt.Sprintf("%x", rawAuthHash)
-	if rows, err := api.db.Queries.CreateUser(api.db.Ctx, database.CreateUserParams{
+	if rows, err := api.db.Queries.CreateUser(c, database.CreateUserParams{
 		ID:       username,
 		Pass:     &hashedPassword,
 		AuthHash: &authHash,
@@ -242,7 +243,7 @@ func (api *API) appAuthRegister(c *gin.Context) {
 	}
 
 	// Get user
-	user, err := api.db.Queries.GetUser(api.db.Ctx, username)
+	user, err := api.db.Queries.GetUser(c, username)
 	if err != nil {
 		log.Error("GetUser DB Error:", err)
 		templateVars["Error"] = "Registration Disabled or User Already Exists"
@@ -312,7 +313,7 @@ func (api *API) koAuthRegister(c *gin.Context) {
 	}
 
 	// Get current users
-	currentUsers, err := api.db.Queries.GetUsers(api.db.Ctx)
+	currentUsers, err := api.db.Queries.GetUsers(c)
 	if err != nil {
 		log.Error("Failed to check all users: ", err)
 		apiErrorPage(c, http.StatusBadRequest, "Failed to Create User")
@@ -327,7 +328,7 @@ func (api *API) koAuthRegister(c *gin.Context) {
 
 	// Create user
 	authHash := fmt.Sprintf("%x", rawAuthHash)
-	if rows, err := api.db.Queries.CreateUser(api.db.Ctx, database.CreateUserParams{
+	if rows, err := api.db.Queries.CreateUser(c, database.CreateUserParams{
 		ID:       rUser.Username,
 		Pass:     &hashedPassword,
 		AuthHash: &authHash,
@@ -347,7 +348,7 @@ func (api *API) koAuthRegister(c *gin.Context) {
 	})
 }
 
-func (api *API) getSession(session sessions.Session) (auth authData, ok bool) {
+func (api *API) getSession(ctx context.Context, session sessions.Session) (auth authData, ok bool) {
 	// Get Session
 	authorizedUser := session.Get("authorizedUser")
 	isAdmin := session.Get("isAdmin")
@@ -365,7 +366,7 @@ func (api *API) getSession(session sessions.Session) (auth authData, ok bool) {
 	}
 
 	// Validate Auth Hash
-	correctAuthHash, err := api.getUserAuthHash(auth.UserName)
+	correctAuthHash, err := api.getUserAuthHash(ctx, auth.UserName)
 	if err != nil || correctAuthHash != auth.AuthHash {
 		return
 	}
@@ -393,14 +394,14 @@ func (api *API) setSession(session sessions.Session, auth authData) error {
 	return session.Save()
 }
 
-func (api *API) getUserAuthHash(username string) (string, error) {
+func (api *API) getUserAuthHash(ctx context.Context, username string) (string, error) {
 	// Return Cache
 	if api.userAuthCache[username] != "" {
 		return api.userAuthCache[username], nil
 	}
 
 	// Get DB
-	user, err := api.db.Queries.GetUser(api.db.Ctx, username)
+	user, err := api.db.Queries.GetUser(ctx, username)
 	if err != nil {
 		log.Error("GetUser DB Error:", err)
 		return "", err
@@ -412,7 +413,7 @@ func (api *API) getUserAuthHash(username string) (string, error) {
 	return api.userAuthCache[username], nil
 }
 
-func (api *API) rotateAllAuthHashes() error {
+func (api *API) rotateAllAuthHashes(ctx context.Context) error {
 	// Do Transaction
 	tx, err := api.db.DB.Begin()
 	if err != nil {
@@ -428,7 +429,7 @@ func (api *API) rotateAllAuthHashes() error {
 	}()
 	qtx := api.db.Queries.WithTx(tx)
 
-	users, err := qtx.GetUsers(api.db.Ctx)
+	users, err := qtx.GetUsers(ctx)
 	if err != nil {
 		return err
 	}
@@ -444,7 +445,7 @@ func (api *API) rotateAllAuthHashes() error {
 
 		// Update User
 		authHash := fmt.Sprintf("%x", rawAuthHash)
-		if _, err = qtx.UpdateUser(api.db.Ctx, database.UpdateUserParams{
+		if _, err = qtx.UpdateUser(ctx, database.UpdateUserParams{
 			UserID:   user.ID,
 			AuthHash: &authHash,
 			Admin:    user.Admin,

api/common.go

@@ -22,7 +22,7 @@ func (api *API) createDownloadDocumentHandler(errorFunc func(*gin.Context, int,
 		}
 
 		// Get Document
-		document, err := api.db.Queries.GetDocument(api.db.Ctx, rDoc.DocumentID)
+		document, err := api.db.Queries.GetDocument(c, rDoc.DocumentID)
 		if err != nil {
 			log.Error("GetDocument DB Error:", err)
 			errorFunc(c, http.StatusBadRequest, "Unknown Document")
@@ -68,7 +68,7 @@ func (api *API) createGetCoverHandler(errorFunc func(*gin.Context, int, string))
 		}
 
 		// Validate Document Exists in DB
-		document, err := api.db.Queries.GetDocument(api.db.Ctx, rDoc.DocumentID)
+		document, err := api.db.Queries.GetDocument(c, rDoc.DocumentID)
 		if err != nil {
 			log.Error("GetDocument DB Error:", err)
 			errorFunc(c, http.StatusInternalServerError, fmt.Sprintf("GetDocument DB Error: %v", err))
@@ -117,7 +117,7 @@ func (api *API) createGetCoverHandler(errorFunc func(*gin.Context, int, string))
 			}
 
 			// Store First Metadata Result
-			if _, err = api.db.Queries.AddMetadata(api.db.Ctx, database.AddMetadataParams{
+			if _, err = api.db.Queries.AddMetadata(c, database.AddMetadataParams{
 				DocumentID:  document.ID,
 				Title:       firstResult.Title,
 				Author:      firstResult.Author,
@@ -132,7 +132,7 @@ func (api *API) createGetCoverHandler(errorFunc func(*gin.Context, int, string))
 		}
 
 		// Upsert Document
-		if _, err = api.db.Queries.UpsertDocument(api.db.Ctx, database.UpsertDocumentParams{
+		if _, err = api.db.Queries.UpsertDocument(c, database.UpsertDocumentParams{
 			ID:        document.ID,
 			Coverfile: &coverFile,
 		}); err != nil {

api/ko-routes.go

@@ -72,7 +72,7 @@ type requestDocumentID struct {
 }
 
 func (api *API) koAuthorizeUser(c *gin.Context) {
-	c.JSON(200, gin.H{
+	koJSON(c, 200, gin.H{
 		"authorized": "OK",
 	})
 }
@@ -91,7 +91,7 @@ func (api *API) koSetProgress(c *gin.Context) {
 	}
 
 	// Upsert Device
-	if _, err := api.db.Queries.UpsertDevice(api.db.Ctx, database.UpsertDeviceParams{
+	if _, err := api.db.Queries.UpsertDevice(c, database.UpsertDeviceParams{
 		ID:         rPosition.DeviceID,
 		UserID:     auth.UserName,
 		DeviceName: rPosition.Device,
@@ -101,14 +101,14 @@ func (api *API) koSetProgress(c *gin.Context) {
 	}
 
 	// Upsert Document
-	if _, err := api.db.Queries.UpsertDocument(api.db.Ctx, database.UpsertDocumentParams{
+	if _, err := api.db.Queries.UpsertDocument(c, database.UpsertDocumentParams{
 		ID: rPosition.DocumentID,
 	}); err != nil {
 		log.Error("UpsertDocument DB Error:", err)
 	}
 
 	// Create or Replace Progress
-	progress, err := api.db.Queries.UpdateProgress(api.db.Ctx, database.UpdateProgressParams{
+	progress, err := api.db.Queries.UpdateProgress(c, database.UpdateProgressParams{
 		Percentage: rPosition.Percentage,
 		DocumentID: rPosition.DocumentID,
 		DeviceID:   rPosition.DeviceID,
@@ -121,7 +121,7 @@ func (api *API) koSetProgress(c *gin.Context) {
 		return
 	}
 
-	c.JSON(http.StatusOK, gin.H{
+	koJSON(c, http.StatusOK, gin.H{
 		"document":  progress.DocumentID,
 		"timestamp": progress.CreatedAt,
 	})
@@ -140,14 +140,14 @@ func (api *API) koGetProgress(c *gin.Context) {
 		return
 	}
 
-	progress, err := api.db.Queries.GetDocumentProgress(api.db.Ctx, database.GetDocumentProgressParams{
+	progress, err := api.db.Queries.GetDocumentProgress(c, database.GetDocumentProgressParams{
 		DocumentID: rDocID.DocumentID,
 		UserID:     auth.UserName,
 	})
 
 	if err == sql.ErrNoRows {
 		// Not Found
-		c.JSON(http.StatusOK, gin.H{})
+		koJSON(c, http.StatusOK, gin.H{})
 		return
 	} else if err != nil {
 		log.Error("GetDocumentProgress DB Error:", err)
@@ -155,7 +155,7 @@ func (api *API) koGetProgress(c *gin.Context) {
 		return
 	}
 
-	c.JSON(http.StatusOK, gin.H{
+	koJSON(c, http.StatusOK, gin.H{
 		"document":   progress.DocumentID,
 		"percentage": progress.Percentage,
 		"progress":   progress.Progress,
@@ -202,7 +202,7 @@ func (api *API) koAddActivities(c *gin.Context) {
 
 	// Upsert Documents
 	for _, doc := range allDocuments {
-		if _, err := qtx.UpsertDocument(api.db.Ctx, database.UpsertDocumentParams{
+		if _, err := qtx.UpsertDocument(c, database.UpsertDocumentParams{
 			ID: doc,
 		}); err != nil {
 			log.Error("UpsertDocument DB Error:", err)
@@ -212,7 +212,7 @@ func (api *API) koAddActivities(c *gin.Context) {
 	}
 
 	// Upsert Device
-	if _, err = qtx.UpsertDevice(api.db.Ctx, database.UpsertDeviceParams{
+	if _, err = qtx.UpsertDevice(c, database.UpsertDeviceParams{
 		ID:         rActivity.DeviceID,
 		UserID:     auth.UserName,
 		DeviceName: rActivity.Device,
@@ -225,7 +225,7 @@ func (api *API) koAddActivities(c *gin.Context) {
 
 	// Add All Activity
 	for _, item := range rActivity.Activity {
-		if _, err := qtx.AddActivity(api.db.Ctx, database.AddActivityParams{
+		if _, err := qtx.AddActivity(c, database.AddActivityParams{
 			UserID:          auth.UserName,
 			DocumentID:      item.DocumentID,
 			DeviceID:        rActivity.DeviceID,
@@ -247,7 +247,7 @@ func (api *API) koAddActivities(c *gin.Context) {
 		return
 	}
 
-	c.JSON(http.StatusOK, gin.H{
+	koJSON(c, http.StatusOK, gin.H{
 		"added": len(rActivity.Activity),
 	})
 }
@@ -266,7 +266,7 @@ func (api *API) koCheckActivitySync(c *gin.Context) {
 	}
 
 	// Upsert Device
-	if _, err := api.db.Queries.UpsertDevice(api.db.Ctx, database.UpsertDeviceParams{
+	if _, err := api.db.Queries.UpsertDevice(c, database.UpsertDeviceParams{
 		ID:         rCheckActivity.DeviceID,
 		UserID:     auth.UserName,
 		DeviceName: rCheckActivity.Device,
@@ -278,7 +278,7 @@ func (api *API) koCheckActivitySync(c *gin.Context) {
 	}
 
 	// Get Last Device Activity
-	lastActivity, err := api.db.Queries.GetLastActivity(api.db.Ctx, database.GetLastActivityParams{
+	lastActivity, err := api.db.Queries.GetLastActivity(c, database.GetLastActivityParams{
 		UserID:   auth.UserName,
 		DeviceID: rCheckActivity.DeviceID,
 	})
@@ -298,7 +298,7 @@ func (api *API) koCheckActivitySync(c *gin.Context) {
 		return
 	}
 
-	c.JSON(http.StatusOK, gin.H{
+	koJSON(c, http.StatusOK, gin.H{
 		"last_sync": parsedTime.Unix(),
 	})
 }
@@ -329,7 +329,7 @@ func (api *API) koAddDocuments(c *gin.Context) {
 
 	// Upsert Documents
 	for _, doc := range rNewDocs.Documents {
-		_, err := qtx.UpsertDocument(api.db.Ctx, database.UpsertDocumentParams{
+		_, err := qtx.UpsertDocument(c, database.UpsertDocumentParams{
 			ID:          doc.ID,
 			Title:       api.sanitizeInput(doc.Title),
 			Author:      api.sanitizeInput(doc.Author),
@@ -352,7 +352,7 @@ func (api *API) koAddDocuments(c *gin.Context) {
 		return
 	}
 
-	c.JSON(http.StatusOK, gin.H{
+	koJSON(c, http.StatusOK, gin.H{
 		"changed": len(rNewDocs.Documents),
 	})
 }
@@ -371,7 +371,7 @@ func (api *API) koCheckDocumentsSync(c *gin.Context) {
 	}
 
 	// Upsert Device
-	_, err := api.db.Queries.UpsertDevice(api.db.Ctx, database.UpsertDeviceParams{
+	_, err := api.db.Queries.UpsertDevice(c, database.UpsertDeviceParams{
 		ID:         rCheckDocs.DeviceID,
 		UserID:     auth.UserName,
 		DeviceName: rCheckDocs.Device,
@@ -384,7 +384,7 @@ func (api *API) koCheckDocumentsSync(c *gin.Context) {
 	}
 
 	// Get Missing Documents
-	missingDocs, err := api.db.Queries.GetMissingDocuments(api.db.Ctx, rCheckDocs.Have)
+	missingDocs, err := api.db.Queries.GetMissingDocuments(c, rCheckDocs.Have)
 	if err != nil {
 		log.Error("GetMissingDocuments DB Error", err)
 		apiErrorPage(c, http.StatusBadRequest, "Invalid Request")
@@ -392,7 +392,7 @@ func (api *API) koCheckDocumentsSync(c *gin.Context) {
 	}
 
 	// Get Deleted Documents
-	deletedDocIDs, err := api.db.Queries.GetDeletedDocuments(api.db.Ctx, rCheckDocs.Have)
+	deletedDocIDs, err := api.db.Queries.GetDeletedDocuments(c, rCheckDocs.Have)
 	if err != nil {
 		log.Error("GetDeletedDocuments DB Error", err)
 		apiErrorPage(c, http.StatusBadRequest, "Invalid Request")
@@ -407,7 +407,7 @@ func (api *API) koCheckDocumentsSync(c *gin.Context) {
 		return
 	}
 
-	wantedDocs, err := api.db.Queries.GetWantedDocuments(api.db.Ctx, string(jsonHaves))
+	wantedDocs, err := api.db.Queries.GetWantedDocuments(c, string(jsonHaves))
 	if err != nil {
 		log.Error("GetWantedDocuments DB Error", err)
 		apiErrorPage(c, http.StatusBadRequest, "Invalid Request")
@@ -447,7 +447,7 @@ func (api *API) koCheckDocumentsSync(c *gin.Context) {
 		rCheckDocSync.Delete = deletedDocIDs
 	}
 
-	c.JSON(http.StatusOK, rCheckDocSync)
+	koJSON(c, http.StatusOK, rCheckDocSync)
 }
 
 func (api *API) koUploadExistingDocument(c *gin.Context) {
@@ -467,7 +467,7 @@ func (api *API) koUploadExistingDocument(c *gin.Context) {
 	}
 
 	// Validate Document Exists in DB
-	document, err := api.db.Queries.GetDocument(api.db.Ctx, rDoc.DocumentID)
+	document, err := api.db.Queries.GetDocument(c, rDoc.DocumentID)
 	if err != nil {
 		log.Error("GetDocument DB Error:", err)
 		apiErrorPage(c, http.StatusBadRequest, "Unknown Document")
@@ -522,7 +522,7 @@ func (api *API) koUploadExistingDocument(c *gin.Context) {
 	}
 
 	// Upsert Document
-	if _, err = api.db.Queries.UpsertDocument(api.db.Ctx, database.UpsertDocumentParams{
+	if _, err = api.db.Queries.UpsertDocument(c, database.UpsertDocumentParams{
 		ID:       document.ID,
 		Md5:      metadataInfo.MD5,
 		Words:    metadataInfo.WordCount,
@@ -534,7 +534,7 @@ func (api *API) koUploadExistingDocument(c *gin.Context) {
 		return
 	}
 
-	c.JSON(http.StatusOK, gin.H{
+	koJSON(c, http.StatusOK, gin.H{
 		"status": "ok",
 	})
 }
@@ -589,3 +589,10 @@ func getFileMD5(filePath string) (*string, error) {
 
 	return &fileHash, nil
 }
+
+// koJSON forces koJSON Content-Type to only return `application/json`. This is addressing
+// the following issue: https://github.com/koreader/koreader/issues/13629
+func koJSON(c *gin.Context, code int, obj any) {
+	c.Header("Content-Type", "application/json")
+	c.JSON(code, obj)
+}

api/opds-routes.go

@@ -10,6 +10,7 @@ import (
 	log "github.com/sirupsen/logrus"
 	"reichard.io/antholume/database"
 	"reichard.io/antholume/opds"
+	"reichard.io/antholume/pkg/ptr"
 )
 
 var mimeMapping map[string]string = map[string]string{
@@ -77,11 +78,12 @@ func (api *API) opdsDocuments(c *gin.Context) {
 	}
 
 	// Get Documents
-	documents, err := api.db.Queries.GetDocumentsWithStats(api.db.Ctx, database.GetDocumentsWithStatsParams{
-		UserID: auth.UserName,
-		Query:  query,
-		Offset: (*qParams.Page - 1) * *qParams.Limit,
-		Limit:  *qParams.Limit,
+	documents, err := api.db.Queries.GetDocumentsWithStats(c, database.GetDocumentsWithStatsParams{
+		UserID:  auth.UserName,
+		Query:   query,
+		Deleted: ptr.Of(false),
+		Offset:  (*qParams.Page - 1) * *qParams.Limit,
+		Limit:   *qParams.Limit,
 	})
 	if err != nil {
 		log.Error("GetDocumentsWithStats DB Error:", err)

api/utils.go

@@ -55,6 +55,7 @@ func getTimeZones() []string {
 
 // niceSeconds takes in an int (in seconds) and returns a string readable
 // representation. For example 1928371 -> "22d 7h 39m 31s".
+// Deprecated: Use formatters.FormatDuration
 func niceSeconds(input int64) (result string) {
 	if input == 0 {
 		return "N/A"
@@ -85,6 +86,7 @@ func niceSeconds(input int64) (result string) {
 
 // niceNumbers takes in an int and returns a string representation. For example
 // 19823 -> "19.8k".
+// Deprecated: Use formatters.FormatNumber
 func niceNumbers(input int64) string {
 	if input == 0 {
 		return "0"

assets/reader/index.htm

@@ -82,13 +82,30 @@
         id="top-bar"
         class="transition-all duration-200 absolute z-10 bg-gray-100 dark:bg-gray-800 w-full px-2"
       >
-        <div class="w-full h-32 flex items-center justify-around relative">
-          <div class="text-gray-500 absolute top-6 left-4 flex flex-col gap-4">
-            <a href="#">
+        <div class="max-h-[75vh] w-full flex flex-col items-center justify-around relative dark:text-white">
+	  <div class="h-32">
+            <div class="text-gray-500 absolute top-6 left-4 flex flex-col gap-4">
+              <a href="#">
+                <svg
+                  width="32"
+                  height="32"
+                  class="cursor-pointer hover:text-gray-800 dark:hover:text-gray-100"
+                  viewBox="0 0 24 24"
+                  fill="currentColor"
+                  xmlns="http://www.w3.org/2000/svg"
+                >
+                  <path
+                    fill-rule="evenodd"
+                    clip-rule="evenodd"
+                    d="M20.5355 3.46447C19.0711 2 16.714 2 12 2C7.28595 2 4.92893 2 3.46447 3.46447C2 4.92893 2 7.28595 2 12C2 16.714 2 19.0711 3.46447 20.5355C4.92893 22 7.28595 22 12 22C16.714 22 19.0711 22 20.5355 20.5355C22 19.0711 22 16.714 22 12C22 7.28595 22 4.92893 20.5355 3.46447ZM14.0303 8.46967C14.3232 8.76256 14.3232 9.23744 14.0303 9.53033L11.5607 12L14.0303 14.4697C14.3232 14.7626 14.3232 15.2374 14.0303 15.5303C13.7374 15.8232 13.2626 15.8232 12.9697 15.5303L9.96967 12.5303C9.82902 12.3897 9.75 12.1989 9.75 12C9.75 11.8011 9.82902 11.6103 9.96967 11.4697L12.9697 8.46967C13.2626 8.17678 13.7374 8.17678 14.0303 8.46967Z"
+                  />
+                </svg>
+              </a>
+
               <svg
                 width="32"
                 height="32"
-                class="cursor-pointer hover:text-gray-800 dark:hover:text-gray-100"
+                class="cursor-pointer hover:text-gray-800 dark:hover:text-gray-100 close-top-bar"
                 viewBox="0 0 24 24"
                 fill="currentColor"
                 xmlns="http://www.w3.org/2000/svg"
@@ -96,61 +113,47 @@
                 <path
                   fill-rule="evenodd"
                   clip-rule="evenodd"
-                  d="M20.5355 3.46447C19.0711 2 16.714 2 12 2C7.28595 2 4.92893 2 3.46447 3.46447C2 4.92893 2 7.28595 2 12C2 16.714 2 19.0711 3.46447 20.5355C4.92893 22 7.28595 22 12 22C16.714 22 19.0711 22 20.5355 20.5355C22 19.0711 22 16.714 22 12C22 7.28595 22 4.92893 20.5355 3.46447ZM14.0303 8.46967C14.3232 8.76256 14.3232 9.23744 14.0303 9.53033L11.5607 12L14.0303 14.4697C14.3232 14.7626 14.3232 15.2374 14.0303 15.5303C13.7374 15.8232 13.2626 15.8232 12.9697 15.5303L9.96967 12.5303C9.82902 12.3897 9.75 12.1989 9.75 12C9.75 11.8011 9.82902 11.6103 9.96967 11.4697L12.9697 8.46967C13.2626 8.17678 13.7374 8.17678 14.0303 8.46967Z"
+                  d="M12 22C7.28595 22 4.92893 22 3.46447 20.5355C2 19.0711 2 16.714 2 12C2 7.28595 2 4.92893 3.46447 3.46447C4.92893 2 7.28595 2 12 2C16.714 2 19.0711 2 20.5355 3.46447C22 4.92893 22 7.28595 22 12C22 16.714 22 19.0711 20.5355 20.5355C19.0711 22 16.714 22 12 22ZM8.96965 8.96967C9.26254 8.67678 9.73742 8.67678 10.0303 8.96967L12 10.9394L13.9696 8.96969C14.2625 8.6768 14.7374 8.6768 15.0303 8.96969C15.3232 9.26258 15.3232 9.73746 15.0303 10.0303L13.0606 12L15.0303 13.9697C15.3232 14.2625 15.3232 14.7374 15.0303 15.0303C14.7374 15.3232 14.2625 15.3232 13.9696 15.0303L12 13.0607L10.0303 15.0303C9.73744 15.3232 9.26256 15.3232 8.96967 15.0303C8.67678 14.7374 8.67678 14.2626 8.96967 13.9697L10.9393 12L8.96965 10.0303C8.67676 9.73744 8.67676 9.26256 8.96965 8.96967Z"
                 />
               </svg>
-            </a>
-
-            <svg
-              width="32"
-              height="32"
-              class="cursor-pointer hover:text-gray-800 dark:hover:text-gray-100 close-top-bar"
-              viewBox="0 0 24 24"
-              fill="currentColor"
-              xmlns="http://www.w3.org/2000/svg"
-            >
-              <path
-                fill-rule="evenodd"
-                clip-rule="evenodd"
-                d="M12 22C7.28595 22 4.92893 22 3.46447 20.5355C2 19.0711 2 16.714 2 12C2 7.28595 2 4.92893 3.46447 3.46447C4.92893 2 7.28595 2 12 2C16.714 2 19.0711 2 20.5355 3.46447C22 4.92893 22 7.28595 22 12C22 16.714 22 19.0711 20.5355 20.5355C19.0711 22 16.714 22 12 22ZM8.96965 8.96967C9.26254 8.67678 9.73742 8.67678 10.0303 8.96967L12 10.9394L13.9696 8.96969C14.2625 8.6768 14.7374 8.6768 15.0303 8.96969C15.3232 9.26258 15.3232 9.73746 15.0303 10.0303L13.0606 12L15.0303 13.9697C15.3232 14.2625 15.3232 14.7374 15.0303 15.0303C14.7374 15.3232 14.2625 15.3232 13.9696 15.0303L12 13.0607L10.0303 15.0303C9.73744 15.3232 9.26256 15.3232 8.96967 15.0303C8.67678 14.7374 8.67678 14.2626 8.96967 13.9697L10.9393 12L8.96965 10.0303C8.67676 9.73744 8.67676 9.26256 8.96965 8.96967Z"
-              />
-            </svg>
-          </div>
-
-          <div class="flex gap-10 h-full p-4 pl-14 rounded">
-            <div class="h-full my-auto relative">
-              <a href="#">
-                <img
-                  class="rounded object-cover h-full"
-                  src="/assets/images/no-cover.jpg"
-                />
-              </a>
             </div>
-            <div class="flex gap-7 justify-around dark:text-white text-sm">
-              <div class="flex flex-col gap-4">
-                <div class="inline-flex shrink-0 items-center">
-                  <div>
-                    <p class="text-gray-400">Title</p>
-                    <p
-                      class="font-medium whitespace-nowrap text-ellipsis overflow-hidden max-w-[50dvw]"
-                    >
-                      "N/A"
-                    </p>
+
+            <div class="flex gap-10 h-full p-4 pl-14 rounded">
+              <div class="h-full my-auto relative">
+                <a href="#">
+                  <img
+                    class="rounded object-cover h-full"
+                    src="/assets/images/no-cover.jpg"
+                  />
+                </a>
+              </div>
+              <div class="flex gap-7 justify-around dark:text-white text-sm">
+                <div class="flex flex-col gap-4">
+                  <div class="inline-flex shrink-0 items-center">
+                    <div>
+                      <p class="text-gray-400">Title</p>
+                      <p
+                        class="font-medium whitespace-nowrap text-ellipsis overflow-hidden max-w-[50dvw]"
+                      >
+                        "N/A"
+                      </p>
+                    </div>
                   </div>
-                </div>
-                <div class="inline-flex shrink-0 items-center">
-                  <div>
-                    <p class="text-gray-400">Author</p>
-                    <p
-                      class="font-medium whitespace-nowrap text-ellipsis overflow-hidden max-w-[50dvw]"
-                    >
-                      "N/A"
-                    </p>
+                  <div class="inline-flex shrink-0 items-center">
+                    <div>
+                      <p class="text-gray-400">Author</p>
+                      <p
+                        class="font-medium whitespace-nowrap text-ellipsis overflow-hidden max-w-[50dvw]"
+                      >
+                        "N/A"
+                      </p>
+                    </div>
                   </div>
                 </div>
               </div>
             </div>
-          </div>
+	  </div>
+	  <div id="toc" class="w-full text-center max-h-[50%] overflow-scroll no-scrollbar"></div>
         </div>
       </div>
 

assets/reader/index.js

@@ -66,6 +66,56 @@ function populateMetadata(data) {
   authorEl.innerText = data.author;
 }
 
+/**
+ * Populate the Table of Contents
+ **/
+function populateTOC() {
+  if (!currentReader.book.navigation.toc) {
+    console.warn("[populateTOC] No TOC");
+    return;
+  }
+
+  let tocEl = document.querySelector("#toc");
+  if (!tocEl) {
+    console.warn("[populateTOC] No TOC Element");
+    return;
+  }
+
+  // Parse the Table of Contents
+  let parsedTOC = currentReader.book.navigation.toc.reduce((agg, item) => {
+    let sectionTitle = item.label.trim();
+    agg.push({ title: sectionTitle, href: item.href });
+    if (item.subitems.length == 0) {
+      return agg;
+    }
+
+    let allSubSections = item.subitems.map(item => {
+      let itemTitle = item.label.trim();
+      if (sectionTitle != "") {
+	itemTitle = sectionTitle + " - " + item.label.trim();
+      }
+      return { title: itemTitle, href: item.href };
+    });
+    agg.push(...allSubSections);
+
+    return agg;
+  }, [])
+
+  // Add Table of Contents to DOM
+  let listEl = document.createElement("ul");
+  listEl.classList.add("m-4")
+  parsedTOC.forEach(item => {
+    let listItem = document.createElement("li");
+    listItem.style.cursor = "pointer";
+    listItem.addEventListener("click", () => {
+      currentReader.rendition.display(item.href);
+    });
+    listItem.textContent = item.title;
+    listEl.appendChild(listItem);
+  });
+  tocEl.appendChild(listEl);
+}
+
 /**
  * This is the main reader class. All functionality is wrapped in this class.
  * Responsible for handling gesture / clicks, flushing progress & activity,
@@ -439,6 +489,7 @@ class EBookReader {
     // ------------------------------------------------ //
     // ----------------- Swipe Helpers ---------------- //
     // ------------------------------------------------ //
+    let disablePagination = false;
     let touchStartX,
       touchStartY,
       touchEndX,
@@ -459,25 +510,38 @@ class EBookReader {
       }
 
       // Swipe Left
-      if (touchEndX + drasticity < touchStartX) {
+      if (!disablePagination && touchEndX + drasticity < touchStartX) {
         nextPage();
       }
 
       // Swipe Right
-      if (touchEndX - drasticity > touchStartX) {
+      if (!disablePagination && touchEndX - drasticity > touchStartX) {
         prevPage();
       }
     }
 
     function handleSwipeDown() {
-      if (bottomBar.classList.contains("bottom-0"))
+      if (bottomBar.classList.contains("bottom-0")) {
         bottomBar.classList.remove("bottom-0");
-      else topBar.classList.add("top-0");
+	disablePagination = false;
+      } else {
+	topBar.classList.add("top-0");
+	populateTOC()
+	disablePagination = true;
+      }
     }
 
     function handleSwipeUp() {
-      if (topBar.classList.contains("top-0")) topBar.classList.remove("top-0");
-      else bottomBar.classList.add("bottom-0");
+      if (topBar.classList.contains("top-0")) {
+	topBar.classList.remove("top-0");
+	disablePagination = false;
+
+	const tocEl = document.querySelector("#toc");
+	if (tocEl) tocEl.innerHTML = "";
+      } else {
+	bottomBar.classList.add("bottom-0");
+	disablePagination = true;
+      }
     }
 
     this.rendition.hooks.render.register(function (doc, data) {
@@ -523,8 +587,8 @@ class EBookReader {
           // Handle Event
           if (yCoord < top) handleSwipeDown();
           else if (yCoord > bottom) handleSwipeUp();
-          else if (xCoord < left) prevPage();
-          else if (xCoord > right) nextPage();
+          else if (!disablePagination && xCoord < left) prevPage();
+          else if (!disablePagination && xCoord > right) nextPage();
           else {
             bottomBar.classList.remove("bottom-0");
             topBar.classList.remove("top-0");
@@ -670,6 +734,9 @@ class EBookReader {
     // Close Top Bar
     document.querySelector(".close-top-bar").addEventListener("click", () => {
       topBar.classList.remove("top-0");
+
+      const tocEl = document.querySelector("#toc");
+      if (tocEl) tocEl.innerHTML = "";
     });
   }
 
@@ -949,10 +1016,16 @@ class EBookReader {
    **/
   async getXPathFromCFI(cfi) {
     // Get DocFragment (Spine Index)
-    let startCFI = cfi.replace("epubcfi(", "");
+    let cfiBaseMatch = cfi.match(/\(([^!]+)/);
+    if (!cfiBaseMatch) {
+      console.error("[getXPathFromCFI] No CFI Match");
+      return {};
+    }
+    let startCFI = cfiBaseMatch[1];
+
     let docFragmentIndex =
       this.book.spine.spineItems.find((item) =>
-        startCFI.startsWith(item.cfiBase),
+        item.cfiBase == startCFI
       ).index + 1;
 
     // Base Progress
@@ -1029,10 +1102,6 @@ class EBookReader {
       return {};
     }
 
-    // Match Item Index
-    let indexMatch = xpath.match(/\.(\d+)$/);
-    let itemIndex = indexMatch ? parseInt(indexMatch[1]) : 0;
-
     // Get Spine Item
     let spinePosition = parseInt(fragMatch[1]) - 1;
     let sectionItem = this.book.spine.get(spinePosition);
@@ -1124,6 +1193,11 @@ class EBookReader {
     let element = docSearch.iterateNext() || derivedSelectorElement;
     let cfi = sectionItem.cfiFromElement(element);
 
+    // Hack - epub.js crashes sometimes when its a bare section with no element
+    // so just return the first.
+    if (cfi.endsWith("!/)"))
+      cfi = cfi.slice(0, -1) + "0)"
+
     return { cfi, element };
   }
 
@@ -1271,14 +1345,3 @@ class EBookReader {
 }
 
 document.addEventListener("DOMContentLoaded", initReader);
-
-// WIP
-async function getTOC() {
-  let toc = currentReader.book.navigation.toc;
-
-  // Alternatively:
-  // let nav = await currentReader.book.loaded.navigation;
-  // let toc = nav.toc;
-
-  currentReader.rendition.display(nav.toc[10].href);
-}

database/db.go

@@ -1,6 +1,6 @@
 // Code generated by sqlc. DO NOT EDIT.
 // versions:
-//   sqlc v1.26.0
+//   sqlc v1.29.0
 
 package database
 

database/document_user_statistics.sql

@@ -0,0 +1,151 @@
+WITH grouped_activity AS (
+    SELECT
+        ga.user_id,
+        ga.document_id,
+        MAX(ga.created_at) AS created_at,
+        MAX(ga.start_time) AS start_time,
+        MIN(ga.start_percentage) AS start_percentage,
+        MAX(ga.end_percentage) AS end_percentage,
+
+        -- Total Duration & Percentage
+        SUM(ga.duration) AS total_time_seconds,
+        SUM(ga.end_percentage - ga.start_percentage) AS total_read_percentage,
+
+        -- Yearly Duration
+        SUM(
+            CASE
+                WHEN
+                    ga.start_time >= DATE('now', '-1 year')
+                    THEN ga.duration
+                ELSE 0
+            END
+        )
+            AS yearly_time_seconds,
+
+        -- Yearly Percentage
+        SUM(
+            CASE
+                WHEN
+                    ga.start_time >= DATE('now', '-1 year')
+                    THEN ga.end_percentage - ga.start_percentage
+                ELSE 0
+            END
+        )
+            AS yearly_read_percentage,
+
+        -- Monthly Duration
+        SUM(
+            CASE
+                WHEN
+                    ga.start_time >= DATE('now', '-1 month')
+                    THEN ga.duration
+                ELSE 0
+            END
+        )
+            AS monthly_time_seconds,
+
+        -- Monthly Percentage
+        SUM(
+            CASE
+                WHEN
+                    ga.start_time >= DATE('now', '-1 month')
+                    THEN ga.end_percentage - ga.start_percentage
+                ELSE 0
+            END
+        )
+            AS monthly_read_percentage,
+
+        -- Weekly Duration
+        SUM(
+            CASE
+                WHEN
+                    ga.start_time >= DATE('now', '-7 days')
+                    THEN ga.duration
+                ELSE 0
+            END
+        )
+            AS weekly_time_seconds,
+
+        -- Weekly Percentage
+        SUM(
+            CASE
+                WHEN
+                    ga.start_time >= DATE('now', '-7 days')
+                    THEN ga.end_percentage - ga.start_percentage
+                ELSE 0
+            END
+        )
+            AS weekly_read_percentage
+
+    FROM activity AS ga
+    GROUP BY ga.user_id, ga.document_id
+),
+
+current_progress AS (
+    SELECT
+        user_id,
+        document_id,
+        COALESCE((
+            SELECT dp.percentage
+            FROM document_progress AS dp
+            WHERE
+                dp.user_id = iga.user_id
+                AND dp.document_id = iga.document_id
+            ORDER BY dp.created_at DESC
+            LIMIT 1
+        ), end_percentage) AS percentage
+    FROM grouped_activity AS iga
+)
+
+INSERT INTO document_user_statistics
+SELECT
+    ga.document_id,
+    ga.user_id,
+    cp.percentage,
+    MAX(ga.start_time) AS last_read,
+    MAX(ga.created_at) AS last_seen,
+    SUM(ga.total_read_percentage) AS read_percentage,
+
+    -- All Time WPM
+    SUM(ga.total_time_seconds) AS total_time_seconds,
+    (CAST(COALESCE(d.words, 0.0) AS REAL) * SUM(ga.total_read_percentage))
+        AS total_words_read,
+    (CAST(COALESCE(d.words, 0.0) AS REAL) * SUM(ga.total_read_percentage))
+    / (SUM(ga.total_time_seconds) / 60.0) AS total_wpm,
+
+    -- Yearly WPM
+    ga.yearly_time_seconds,
+    CAST(COALESCE(d.words, 0.0) AS REAL) * ga.yearly_read_percentage
+        AS yearly_words_read,
+    COALESCE(
+        (CAST(COALESCE(d.words, 0.0) AS REAL) * ga.yearly_read_percentage)
+        / (ga.yearly_time_seconds / 60), 0.0)
+        AS yearly_wpm,
+
+    -- Monthly WPM
+    ga.monthly_time_seconds,
+    CAST(COALESCE(d.words, 0.0) AS REAL) * ga.monthly_read_percentage
+        AS monthly_words_read,
+    COALESCE(
+        (CAST(COALESCE(d.words, 0.0) AS REAL) * ga.monthly_read_percentage)
+        / (ga.monthly_time_seconds / 60), 0.0)
+        AS monthly_wpm,
+
+    -- Weekly WPM
+    ga.weekly_time_seconds,
+    CAST(COALESCE(d.words, 0.0) AS REAL) * ga.weekly_read_percentage
+        AS weekly_words_read,
+    COALESCE(
+        (CAST(COALESCE(d.words, 0.0) AS REAL) * ga.weekly_read_percentage)
+        / (ga.weekly_time_seconds / 60), 0.0)
+        AS weekly_wpm
+
+FROM grouped_activity AS ga
+INNER JOIN
+    current_progress AS cp
+    ON ga.user_id = cp.user_id AND ga.document_id = cp.document_id
+INNER JOIN
+    documents AS d
+    ON ga.document_id = d.id
+GROUP BY ga.document_id, ga.user_id
+ORDER BY total_wpm DESC;

database/documents.go

@@ -0,0 +1,27 @@
+package database
+
+import (
+	"context"
+	"fmt"
+
+	"reichard.io/antholume/pkg/ptr"
+	"reichard.io/antholume/pkg/sliceutils"
+)
+
+func (d *DBManager) GetDocument(ctx context.Context, docID, userID string) (*GetDocumentsWithStatsRow, error) {
+	documents, err := d.Queries.GetDocumentsWithStats(ctx, GetDocumentsWithStatsParams{
+		ID:     ptr.Of(docID),
+		UserID: userID,
+		Limit:  1,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	document, found := sliceutils.First(documents)
+	if !found {
+		return nil, fmt.Errorf("document not found: %s", docID)
+	}
+
+	return &document, nil
+}

database/documents_test.go

@@ -1,6 +1,7 @@
 package database
 
 import (
+	"context"
 	"fmt"
 	"testing"
 
@@ -26,7 +27,7 @@ func (suite *DocumentsTestSuite) SetupTest() {
 	suite.dbm = NewMgr(&cfg)
 
 	// Create Document
-	_, err := suite.dbm.Queries.UpsertDocument(suite.dbm.Ctx, UpsertDocumentParams{
+	_, err := suite.dbm.Queries.UpsertDocument(context.Background(), UpsertDocumentParams{
 		ID:     documentID,
 		Title:  &documentTitle,
 		Author: &documentAuthor,
@@ -42,7 +43,7 @@ func (suite *DocumentsTestSuite) SetupTest() {
 //   - 󰊕  (q *Queries) GetDocumentsWithStats
 //   - 󰊕  (q *Queries) GetMissingDocuments
 func (suite *DocumentsTestSuite) TestGetDocument() {
-	doc, err := suite.dbm.Queries.GetDocument(suite.dbm.Ctx, documentID)
+	doc, err := suite.dbm.Queries.GetDocument(context.Background(), documentID)
 	suite.Nil(err, "should have nil err")
 	suite.Equal(documentID, doc.ID, "should have changed the document")
 }
@@ -50,7 +51,7 @@ func (suite *DocumentsTestSuite) TestGetDocument() {
 func (suite *DocumentsTestSuite) TestUpsertDocument() {
 	testDocID := "docid1"
 
-	doc, err := suite.dbm.Queries.UpsertDocument(suite.dbm.Ctx, UpsertDocumentParams{
+	doc, err := suite.dbm.Queries.UpsertDocument(context.Background(), UpsertDocumentParams{
 		ID:     testDocID,
 		Title:  &documentTitle,
 		Author: &documentAuthor,
@@ -63,51 +64,51 @@ func (suite *DocumentsTestSuite) TestUpsertDocument() {
 }
 
 func (suite *DocumentsTestSuite) TestDeleteDocument() {
-	changed, err := suite.dbm.Queries.DeleteDocument(suite.dbm.Ctx, documentID)
+	changed, err := suite.dbm.Queries.DeleteDocument(context.Background(), documentID)
 	suite.Nil(err, "should have nil err")
 	suite.Equal(int64(1), changed, "should have changed the document")
 
-	doc, err := suite.dbm.Queries.GetDocument(suite.dbm.Ctx, documentID)
+	doc, err := suite.dbm.Queries.GetDocument(context.Background(), documentID)
 	suite.Nil(err, "should have nil err")
 	suite.True(doc.Deleted, "should have deleted the document")
 }
 
 func (suite *DocumentsTestSuite) TestGetDeletedDocuments() {
-	changed, err := suite.dbm.Queries.DeleteDocument(suite.dbm.Ctx, documentID)
+	changed, err := suite.dbm.Queries.DeleteDocument(context.Background(), documentID)
 	suite.Nil(err, "should have nil err")
 	suite.Equal(int64(1), changed, "should have changed the document")
 
-	deletedDocs, err := suite.dbm.Queries.GetDeletedDocuments(suite.dbm.Ctx, []string{documentID})
+	deletedDocs, err := suite.dbm.Queries.GetDeletedDocuments(context.Background(), []string{documentID})
 	suite.Nil(err, "should have nil err")
 	suite.Len(deletedDocs, 1, "should have one deleted document")
 }
 
 // TODO - Convert GetWantedDocuments -> (sqlc.slice('document_ids'));
 func (suite *DocumentsTestSuite) TestGetWantedDocuments() {
-	wantedDocs, err := suite.dbm.Queries.GetWantedDocuments(suite.dbm.Ctx, fmt.Sprintf("[\"%s\"]", documentID))
+	wantedDocs, err := suite.dbm.Queries.GetWantedDocuments(context.Background(), fmt.Sprintf("[\"%s\"]", documentID))
 	suite.Nil(err, "should have nil err")
 	suite.Len(wantedDocs, 1, "should have one wanted document")
 }
 
 func (suite *DocumentsTestSuite) TestGetMissingDocuments() {
 	// Create Document
-	_, err := suite.dbm.Queries.UpsertDocument(suite.dbm.Ctx, UpsertDocumentParams{
+	_, err := suite.dbm.Queries.UpsertDocument(context.Background(), UpsertDocumentParams{
 		ID:       documentID,
 		Filepath: &documentFilepath,
 	})
 	suite.NoError(err)
 
-	missingDocs, err := suite.dbm.Queries.GetMissingDocuments(suite.dbm.Ctx, []string{documentID})
+	missingDocs, err := suite.dbm.Queries.GetMissingDocuments(context.Background(), []string{documentID})
 	suite.Nil(err, "should have nil err")
 	suite.Len(missingDocs, 0, "should have no wanted document")
 
-	missingDocs, err = suite.dbm.Queries.GetMissingDocuments(suite.dbm.Ctx, []string{"other"})
+	missingDocs, err = suite.dbm.Queries.GetMissingDocuments(context.Background(), []string{"other"})
 	suite.Nil(err, "should have nil err")
 	suite.Len(missingDocs, 1, "should have one missing document")
 	suite.Equal(documentID, missingDocs[0].ID, "should have missing doc")
 
 	// TODO - https://github.com/sqlc-dev/sqlc/issues/3451
-	// missingDocs, err = suite.dbm.Queries.GetMissingDocuments(suite.dbm.Ctx, []string{})
+	// missingDocs, err = suite.dbm.Queries.GetMissingDocuments(context.Background(), []string{})
 	// suite.Nil(err, "should have nil err")
 	// suite.Len(missingDocs, 1, "should have one missing document")
 	// suite.Equal(documentID, missingDocs[0].ID, "should have missing doc")

database/manager.go

@@ -5,7 +5,6 @@ import (
 	"database/sql"
 	"database/sql/driver"
 	"embed"
-	_ "embed"
 	"errors"
 	"fmt"
 	"path/filepath"
@@ -20,7 +19,6 @@ import (
 
 type DBManager struct {
 	DB      *sql.DB
-	Ctx     context.Context
 	Queries *Queries
 	cfg     *config.Config
 }
@@ -28,8 +26,11 @@ type DBManager struct {
 //go:embed schema.sql
 var ddl string
 
-//go:embed views.sql
-var views string
+//go:embed user_streaks.sql
+var user_streaks string
+
+//go:embed document_user_statistics.sql
+var document_user_statistics string
 
 //go:embed migrations/*
 var migrations embed.FS
@@ -41,17 +42,19 @@ func init() {
 		Deterministic: true,
 		Scalar:        localTime,
 	})
+	sqlite.MustRegisterFunction("LOCAL_DATE", &sqlite.FunctionImpl{
+		NArgs:         2,
+		Deterministic: true,
+		Scalar:        localDate,
+	})
 }
 
 // NewMgr Returns an initialized manager
 func NewMgr(c *config.Config) *DBManager {
 	// Create Manager
-	dbm := &DBManager{
-		Ctx: context.Background(),
-		cfg: c,
-	}
+	dbm := &DBManager{cfg: c}
 
-	if err := dbm.init(); err != nil {
+	if err := dbm.init(context.Background()); err != nil {
 		log.Panic("Unable to init DB")
 	}
 
@@ -59,7 +62,7 @@ func NewMgr(c *config.Config) *DBManager {
 }
 
 // init loads the DB manager
-func (dbm *DBManager) init() error {
+func (dbm *DBManager) init(ctx context.Context) error {
 	// Build DB Location
 	var dbLocation string
 	switch dbm.cfg.DBType {
@@ -104,21 +107,15 @@ func (dbm *DBManager) init() error {
 		return err
 	}
 
-	// Execute views
-	if _, err := dbm.DB.Exec(views, nil); err != nil {
-		log.Panicf("Error executing views: %v", err)
-		return err
-	}
-
 	// Update settings
-	err = dbm.updateSettings()
+	err = dbm.updateSettings(ctx)
 	if err != nil {
 		log.Panicf("Error running DB settings update: %v", err)
 		return err
 	}
 
 	// Cache tables
-	if err := dbm.CacheTempTables(); err != nil {
+	if err := dbm.CacheTempTables(ctx); err != nil {
 		log.Warn("Refreshing temp table cache failed: ", err)
 	}
 
@@ -126,7 +123,7 @@ func (dbm *DBManager) init() error {
 }
 
 // Reload closes the DB & reinits
-func (dbm *DBManager) Reload() error {
+func (dbm *DBManager) Reload(ctx context.Context) error {
 	// Close handle
 	err := dbm.DB.Close()
 	if err != nil {
@@ -134,7 +131,7 @@ func (dbm *DBManager) Reload() error {
 	}
 
 	// Reinit DB
-	if err := dbm.init(); err != nil {
+	if err := dbm.init(ctx); err != nil {
 		return err
 	}
 
@@ -142,23 +139,15 @@ func (dbm *DBManager) Reload() error {
 }
 
 // CacheTempTables clears existing statistics and recalculates
-func (dbm *DBManager) CacheTempTables() error {
+func (dbm *DBManager) CacheTempTables(ctx context.Context) error {
 	start := time.Now()
-	user_streaks_sql := `
-	  DELETE FROM user_streaks;
-	  INSERT INTO user_streaks SELECT * FROM view_user_streaks;
-	`
-	if _, err := dbm.DB.ExecContext(dbm.Ctx, user_streaks_sql); err != nil {
+	if _, err := dbm.DB.ExecContext(ctx, user_streaks); err != nil {
 		return err
 	}
 	log.Debug("Cached 'user_streaks' in: ", time.Since(start))
 
 	start = time.Now()
-	document_statistics_sql := `
-	  DELETE FROM document_user_statistics;
-	  INSERT INTO document_user_statistics SELECT * FROM view_document_user_statistics;
-	`
-	if _, err := dbm.DB.ExecContext(dbm.Ctx, document_statistics_sql); err != nil {
+	if _, err := dbm.DB.ExecContext(ctx, document_user_statistics); err != nil {
 		return err
 	}
 	log.Debug("Cached 'document_user_statistics' in: ", time.Since(start))
@@ -168,7 +157,7 @@ func (dbm *DBManager) CacheTempTables() error {
 
 // updateSettings ensures that we're enforcing foreign keys and enable journal
 // mode.
-func (dbm *DBManager) updateSettings() error {
+func (dbm *DBManager) updateSettings(ctx context.Context) error {
 	// Set SQLite PRAGMA Settings
 	pragmaQuery := `
 		  PRAGMA foreign_keys = ON;
@@ -180,7 +169,7 @@ func (dbm *DBManager) updateSettings() error {
 	}
 
 	// Update Antholume Version in DB
-	if _, err := dbm.Queries.UpdateSettings(dbm.Ctx, UpdateSettingsParams{
+	if _, err := dbm.Queries.UpdateSettings(ctx, UpdateSettingsParams{
 		Name:  "version",
 		Value: dbm.cfg.Version,
 	}); err != nil {
@@ -240,5 +229,30 @@ func localTime(ctx *sqlite.FunctionContext, args []driver.Value) (driver.Value,
 		return nil, errors.New("unable to parse time")
 	}
 
-	return formattedTime.In(timeZone).Format("2006-01-02 15:04:05.000"), nil
+	return formattedTime.In(timeZone).Format(time.RFC3339), nil
+}
+
+// localDate is a custom SQL function that is registered as LOCAL_DATE in the init function
+func localDate(ctx *sqlite.FunctionContext, args []driver.Value) (driver.Value, error) {
+	timeStr, ok := args[0].(string)
+	if !ok {
+		return nil, errors.New("both arguments to TZTime must be strings")
+	}
+
+	timeZoneStr, ok := args[1].(string)
+	if !ok {
+		return nil, errors.New("both arguments to TZTime must be strings")
+	}
+
+	timeZone, err := time.LoadLocation(timeZoneStr)
+	if err != nil {
+		return nil, errors.New("unable to parse timezone")
+	}
+
+	formattedTime, err := time.ParseInLocation(time.RFC3339, timeStr, time.UTC)
+	if err != nil {
+		return nil, errors.New("unable to parse time")
+	}
+
+	return formattedTime.In(timeZone).Format("2006-01-02"), nil
 }

database/manager_test.go

@@ -1,6 +1,7 @@
 package database
 
 import (
+	"context"
 	"fmt"
 	"testing"
 	"time"
@@ -46,7 +47,7 @@ func (suite *DatabaseTestSuite) SetupTest() {
 	// Create User
 	rawAuthHash, _ := utils.GenerateToken(64)
 	authHash := fmt.Sprintf("%x", rawAuthHash)
-	_, err := suite.dbm.Queries.CreateUser(suite.dbm.Ctx, CreateUserParams{
+	_, err := suite.dbm.Queries.CreateUser(context.Background(), CreateUserParams{
 		ID:       userID,
 		Pass:     &userPass,
 		AuthHash: &authHash,
@@ -54,7 +55,7 @@ func (suite *DatabaseTestSuite) SetupTest() {
 	suite.NoError(err)
 
 	// Create Document
-	_, err = suite.dbm.Queries.UpsertDocument(suite.dbm.Ctx, UpsertDocumentParams{
+	_, err = suite.dbm.Queries.UpsertDocument(context.Background(), UpsertDocumentParams{
 		ID:       documentID,
 		Title:    &documentTitle,
 		Author:   &documentAuthor,
@@ -64,7 +65,7 @@ func (suite *DatabaseTestSuite) SetupTest() {
 	suite.NoError(err)
 
 	// Create Device
-	_, err = suite.dbm.Queries.UpsertDevice(suite.dbm.Ctx, UpsertDeviceParams{
+	_, err = suite.dbm.Queries.UpsertDevice(context.Background(), UpsertDeviceParams{
 		ID:         deviceID,
 		UserID:     userID,
 		DeviceName: deviceName,
@@ -80,7 +81,7 @@ func (suite *DatabaseTestSuite) SetupTest() {
 		counter += 1
 
 		// Add Item
-		activity, err := suite.dbm.Queries.AddActivity(suite.dbm.Ctx, AddActivityParams{
+		activity, err := suite.dbm.Queries.AddActivity(context.Background(), AddActivityParams{
 			DocumentID:      documentID,
 			DeviceID:        deviceID,
 			UserID:          userID,
@@ -95,7 +96,7 @@ func (suite *DatabaseTestSuite) SetupTest() {
 	}
 
 	// Initiate Cache
-	err = suite.dbm.CacheTempTables()
+	err = suite.dbm.CacheTempTables(context.Background())
 	suite.NoError(err)
 }
 
@@ -105,7 +106,7 @@ func (suite *DatabaseTestSuite) SetupTest() {
 //   - 󰊕  (q *Queries) UpsertDevice
 func (suite *DatabaseTestSuite) TestDevice() {
 	testDevice := "dev123"
-	device, err := suite.dbm.Queries.UpsertDevice(suite.dbm.Ctx, UpsertDeviceParams{
+	device, err := suite.dbm.Queries.UpsertDevice(context.Background(), UpsertDeviceParams{
 		ID:         testDevice,
 		UserID:     userID,
 		DeviceName: deviceName,
@@ -123,7 +124,7 @@ func (suite *DatabaseTestSuite) TestDevice() {
 //   - 󰊕  (q *Queries) GetLastActivity
 func (suite *DatabaseTestSuite) TestActivity() {
 	// Validate Exists
-	existsRows, err := suite.dbm.Queries.GetActivity(suite.dbm.Ctx, GetActivityParams{
+	existsRows, err := suite.dbm.Queries.GetActivity(context.Background(), GetActivityParams{
 		UserID: userID,
 		Offset: 0,
 		Limit:  50,
@@ -133,7 +134,7 @@ func (suite *DatabaseTestSuite) TestActivity() {
 	suite.Len(existsRows, 10, "should have correct number of rows get activity")
 
 	// Validate Doesn't Exist
-	doesntExistsRows, err := suite.dbm.Queries.GetActivity(suite.dbm.Ctx, GetActivityParams{
+	doesntExistsRows, err := suite.dbm.Queries.GetActivity(context.Background(), GetActivityParams{
 		UserID:     userID,
 		DocumentID: "unknownDoc",
 		DocFilter:  true,
@@ -151,7 +152,7 @@ func (suite *DatabaseTestSuite) TestActivity() {
 //   - 󰊕  (q *Queries) GetDatabaseInfo
 //   - 󰊕  (q *Queries) UpdateSettings
 func (suite *DatabaseTestSuite) TestGetDailyReadStats() {
-	readStats, err := suite.dbm.Queries.GetDailyReadStats(suite.dbm.Ctx, userID)
+	readStats, err := suite.dbm.Queries.GetDailyReadStats(context.Background(), userID)
 
 	suite.Nil(err, "should have nil err")
 	suite.Len(readStats, 30, "should have length of 30")

database/models.go

@@ -1,6 +1,6 @@
 // Code generated by sqlc. DO NOT EDIT.
 // versions:
-//   sqlc v1.26.0
+//   sqlc v1.29.0
 
 package database
 
@@ -62,6 +62,7 @@ type DocumentUserStatistic struct {
 	UserID             string  `json:"user_id"`
 	Percentage         float64 `json:"percentage"`
 	LastRead           string  `json:"last_read"`
+	LastSeen           string  `json:"last_seen"`
 	ReadPercentage     float64 `json:"read_percentage"`
 	TotalTimeSeconds   int64   `json:"total_time_seconds"`
 	TotalWordsRead     int64   `json:"total_words_read"`
@@ -77,7 +78,7 @@ type DocumentUserStatistic struct {
 	WeeklyWpm          float64 `json:"weekly_wpm"`
 }
 
-type Metadatum struct {
+type Metadata struct {
 	ID          int64   `json:"id"`
 	DocumentID  string  `json:"document_id"`
 	Title       *string `json:"title"`
@@ -115,4 +116,8 @@ type UserStreak struct {
 	CurrentStreak          int64  `json:"current_streak"`
 	CurrentStreakStartDate string `json:"current_streak_start_date"`
 	CurrentStreakEndDate   string `json:"current_streak_end_date"`
+	LastTimezone           string `json:"last_timezone"`
+	LastSeen               string `json:"last_seen"`
+	LastRecord             string `json:"last_record"`
+	LastCalculated         string `json:"last_calculated"`
 }

database/query.sql

@@ -67,7 +67,7 @@ WITH filtered_activity AS (
 SELECT
     document_id,
     device_id,
-    CAST(STRFTIME('%Y-%m-%d %H:%M:%S', LOCAL_TIME(activity.start_time, users.timezone)) AS TEXT) AS start_time,
+    LOCAL_TIME(activity.start_time, users.timezone) AS start_time,
     title,
     author,
     duration,
@@ -80,7 +80,7 @@ LEFT JOIN users ON users.id = activity.user_id;
 
 -- name: GetDailyReadStats :many
 WITH RECURSIVE last_30_days AS (
-    SELECT DATE(LOCAL_TIME(STRFTIME('%Y-%m-%dT%H:%M:%SZ', 'now'), timezone)) AS date
+    SELECT LOCAL_DATE(STRFTIME('%Y-%m-%dT%H:%M:%SZ', 'now'), timezone) AS date
     FROM users WHERE users.id = $user_id
     UNION ALL
     SELECT DATE(date, '-1 days')
@@ -99,7 +99,7 @@ filtered_activity AS (
 activity_days AS (
     SELECT
         SUM(duration) AS seconds_read,
-        DATE(LOCAL_TIME(start_time, timezone)) AS day
+        LOCAL_DATE(start_time, timezone) AS day
     FROM filtered_activity AS activity
     LEFT JOIN users ON users.id = activity.user_id
     GROUP BY day
@@ -138,8 +138,8 @@ WHERE id = $device_id LIMIT 1;
 SELECT
     devices.id,
     devices.device_name,
-    CAST(STRFTIME('%Y-%m-%d %H:%M:%S', LOCAL_TIME(devices.created_at, users.timezone)) AS TEXT) AS created_at,
-    CAST(STRFTIME('%Y-%m-%d %H:%M:%S', LOCAL_TIME(devices.last_synced, users.timezone)) AS TEXT) AS last_synced
+    LOCAL_TIME(devices.created_at, users.timezone) AS created_at,
+    LOCAL_TIME(devices.last_synced, users.timezone) AS last_synced
 FROM devices
 JOIN users ON users.id = devices.user_id
 WHERE users.id = $user_id
@@ -163,42 +163,6 @@ ORDER BY
     DESC
 LIMIT 1;
 
--- name: GetDocumentWithStats :one
-SELECT
-    docs.id,
-    docs.title,
-    docs.author,
-    docs.description,
-    docs.isbn10,
-    docs.isbn13,
-    docs.filepath,
-    docs.words,
-
-    CAST(COALESCE(dus.total_wpm, 0.0) AS INTEGER) AS wpm,
-    COALESCE(dus.read_percentage, 0) AS read_percentage,
-    COALESCE(dus.total_time_seconds, 0) AS total_time_seconds,
-    STRFTIME('%Y-%m-%d %H:%M:%S', LOCAL_TIME(COALESCE(dus.last_read, STRFTIME('%Y-%m-%dT%H:%M:%SZ', 0, 'unixepoch')), users.timezone))
-        AS last_read,
-    ROUND(CAST(CASE
-        WHEN dus.percentage IS NULL THEN 0.0
-        WHEN (dus.percentage * 100.0) > 97.0 THEN 100.0
-        ELSE dus.percentage * 100.0
-    END AS REAL), 2) AS percentage,
-    CAST(CASE
-        WHEN dus.total_time_seconds IS NULL THEN 0.0
-        ELSE
-            CAST(dus.total_time_seconds AS REAL)
-            / (dus.read_percentage * 100.0)
-    END AS INTEGER) AS seconds_per_percent
-FROM documents AS docs
-LEFT JOIN users ON users.id = $user_id
-LEFT JOIN
-    document_user_statistics AS dus
-    ON dus.document_id = docs.id AND dus.user_id = $user_id
-WHERE users.id = $user_id
-AND docs.id = $document_id
-LIMIT 1;
-
 -- name: GetDocuments :many
 SELECT * FROM documents
 ORDER BY created_at DESC
@@ -236,26 +200,25 @@ SELECT
         WHEN (dus.percentage * 100.0) > 97.0 THEN 100.0
         ELSE dus.percentage * 100.0
     END AS REAL), 2) AS percentage,
-
-    CASE
+    CAST(CASE
         WHEN dus.total_time_seconds IS NULL THEN 0.0
         ELSE
-            ROUND(
-                CAST(dus.total_time_seconds AS REAL)
-                / (dus.read_percentage * 100.0)
-            )
-    END AS seconds_per_percent
+            CAST(dus.total_time_seconds AS REAL)
+            / (dus.read_percentage * 100.0)
+    END AS INTEGER) AS seconds_per_percent
 FROM documents AS docs
 LEFT JOIN users ON users.id = $user_id
 LEFT JOIN
     document_user_statistics AS dus
     ON dus.document_id = docs.id AND dus.user_id = $user_id
 WHERE
-    docs.deleted = false AND (
-        $query IS NULL OR (
-            docs.title LIKE $query OR
+    (docs.id = sqlc.narg('id') OR $id IS NULL)
+    AND (docs.deleted = sqlc.narg(deleted) OR $deleted IS NULL)
+    AND (
+        (
+            docs.title LIKE sqlc.narg('query') OR
             docs.author LIKE $query
-        )
+        ) OR $query IS NULL
     )
 ORDER BY dus.last_read DESC, docs.created_at DESC
 LIMIT $limit
@@ -283,7 +246,7 @@ SELECT
     ROUND(CAST(progress.percentage AS REAL) * 100, 2) AS percentage,
     progress.document_id,
     progress.user_id,
-    CAST(STRFTIME('%Y-%m-%d %H:%M:%S', LOCAL_TIME(progress.created_at, users.timezone)) AS TEXT) AS created_at
+    LOCAL_TIME(progress.created_at, users.timezone) AS created_at
 FROM document_progress AS progress
 LEFT JOIN users ON progress.user_id = users.id
 LEFT JOIN devices ON progress.device_id = devices.id

database/query.sql.go

@@ -1,6 +1,6 @@
 // Code generated by sqlc. DO NOT EDIT.
 // versions:
-//   sqlc v1.26.0
+//   sqlc v1.29.0
 // source: query.sql
 
 package database
@@ -85,7 +85,7 @@ type AddMetadataParams struct {
 	Isbn13      *string `json:"isbn13"`
 }
 
-func (q *Queries) AddMetadata(ctx context.Context, arg AddMetadataParams) (Metadatum, error) {
+func (q *Queries) AddMetadata(ctx context.Context, arg AddMetadataParams) (Metadata, error) {
 	row := q.db.QueryRowContext(ctx, addMetadata,
 		arg.DocumentID,
 		arg.Title,
@@ -96,7 +96,7 @@ func (q *Queries) AddMetadata(ctx context.Context, arg AddMetadataParams) (Metad
 		arg.Isbn10,
 		arg.Isbn13,
 	)
-	var i Metadatum
+	var i Metadata
 	err := row.Scan(
 		&i.ID,
 		&i.DocumentID,
@@ -193,7 +193,7 @@ WITH filtered_activity AS (
 SELECT
     document_id,
     device_id,
-    CAST(STRFTIME('%Y-%m-%d %H:%M:%S', LOCAL_TIME(activity.start_time, users.timezone)) AS TEXT) AS start_time,
+    LOCAL_TIME(activity.start_time, users.timezone) AS start_time,
     title,
     author,
     duration,
@@ -214,15 +214,15 @@ type GetActivityParams struct {
 }
 
 type GetActivityRow struct {
-	DocumentID      string  `json:"document_id"`
-	DeviceID        string  `json:"device_id"`
-	StartTime       string  `json:"start_time"`
-	Title           *string `json:"title"`
-	Author          *string `json:"author"`
-	Duration        int64   `json:"duration"`
-	StartPercentage float64 `json:"start_percentage"`
-	EndPercentage   float64 `json:"end_percentage"`
-	ReadPercentage  float64 `json:"read_percentage"`
+	DocumentID      string      `json:"document_id"`
+	DeviceID        string      `json:"device_id"`
+	StartTime       interface{} `json:"start_time"`
+	Title           *string     `json:"title"`
+	Author          *string     `json:"author"`
+	Duration        int64       `json:"duration"`
+	StartPercentage float64     `json:"start_percentage"`
+	EndPercentage   float64     `json:"end_percentage"`
+	ReadPercentage  float64     `json:"read_percentage"`
 }
 
 func (q *Queries) GetActivity(ctx context.Context, arg GetActivityParams) ([]GetActivityRow, error) {
@@ -266,7 +266,7 @@ func (q *Queries) GetActivity(ctx context.Context, arg GetActivityParams) ([]Get
 
 const getDailyReadStats = `-- name: GetDailyReadStats :many
 WITH RECURSIVE last_30_days AS (
-    SELECT DATE(LOCAL_TIME(STRFTIME('%Y-%m-%dT%H:%M:%SZ', 'now'), timezone)) AS date
+    SELECT LOCAL_DATE(STRFTIME('%Y-%m-%dT%H:%M:%SZ', 'now'), timezone) AS date
     FROM users WHERE users.id = ?1
     UNION ALL
     SELECT DATE(date, '-1 days')
@@ -285,7 +285,7 @@ filtered_activity AS (
 activity_days AS (
     SELECT
         SUM(duration) AS seconds_read,
-        DATE(LOCAL_TIME(start_time, timezone)) AS day
+        LOCAL_DATE(start_time, timezone) AS day
     FROM filtered_activity AS activity
     LEFT JOIN users ON users.id = activity.user_id
     GROUP BY day
@@ -422,8 +422,8 @@ const getDevices = `-- name: GetDevices :many
 SELECT
     devices.id,
     devices.device_name,
-    CAST(STRFTIME('%Y-%m-%d %H:%M:%S', LOCAL_TIME(devices.created_at, users.timezone)) AS TEXT) AS created_at,
-    CAST(STRFTIME('%Y-%m-%d %H:%M:%S', LOCAL_TIME(devices.last_synced, users.timezone)) AS TEXT) AS last_synced
+    LOCAL_TIME(devices.created_at, users.timezone) AS created_at,
+    LOCAL_TIME(devices.last_synced, users.timezone) AS last_synced
 FROM devices
 JOIN users ON users.id = devices.user_id
 WHERE users.id = ?1
@@ -431,10 +431,10 @@ ORDER BY devices.last_synced DESC
 `
 
 type GetDevicesRow struct {
-	ID         string `json:"id"`
-	DeviceName string `json:"device_name"`
-	CreatedAt  string `json:"created_at"`
-	LastSynced string `json:"last_synced"`
+	ID         string      `json:"id"`
+	DeviceName string      `json:"device_name"`
+	CreatedAt  interface{} `json:"created_at"`
+	LastSynced interface{} `json:"last_synced"`
 }
 
 func (q *Queries) GetDevices(ctx context.Context, userID string) ([]GetDevicesRow, error) {
@@ -543,87 +543,6 @@ func (q *Queries) GetDocumentProgress(ctx context.Context, arg GetDocumentProgre
 	return i, err
 }
 
-const getDocumentWithStats = `-- name: GetDocumentWithStats :one
-SELECT
-    docs.id,
-    docs.title,
-    docs.author,
-    docs.description,
-    docs.isbn10,
-    docs.isbn13,
-    docs.filepath,
-    docs.words,
-
-    CAST(COALESCE(dus.total_wpm, 0.0) AS INTEGER) AS wpm,
-    COALESCE(dus.read_percentage, 0) AS read_percentage,
-    COALESCE(dus.total_time_seconds, 0) AS total_time_seconds,
-    STRFTIME('%Y-%m-%d %H:%M:%S', LOCAL_TIME(COALESCE(dus.last_read, STRFTIME('%Y-%m-%dT%H:%M:%SZ', 0, 'unixepoch')), users.timezone))
-        AS last_read,
-    ROUND(CAST(CASE
-        WHEN dus.percentage IS NULL THEN 0.0
-        WHEN (dus.percentage * 100.0) > 97.0 THEN 100.0
-        ELSE dus.percentage * 100.0
-    END AS REAL), 2) AS percentage,
-    CAST(CASE
-        WHEN dus.total_time_seconds IS NULL THEN 0.0
-        ELSE
-            CAST(dus.total_time_seconds AS REAL)
-            / (dus.read_percentage * 100.0)
-    END AS INTEGER) AS seconds_per_percent
-FROM documents AS docs
-LEFT JOIN users ON users.id = ?1
-LEFT JOIN
-    document_user_statistics AS dus
-    ON dus.document_id = docs.id AND dus.user_id = ?1
-WHERE users.id = ?1
-AND docs.id = ?2
-LIMIT 1
-`
-
-type GetDocumentWithStatsParams struct {
-	UserID     string `json:"user_id"`
-	DocumentID string `json:"document_id"`
-}
-
-type GetDocumentWithStatsRow struct {
-	ID                string      `json:"id"`
-	Title             *string     `json:"title"`
-	Author            *string     `json:"author"`
-	Description       *string     `json:"description"`
-	Isbn10            *string     `json:"isbn10"`
-	Isbn13            *string     `json:"isbn13"`
-	Filepath          *string     `json:"filepath"`
-	Words             *int64      `json:"words"`
-	Wpm               int64       `json:"wpm"`
-	ReadPercentage    float64     `json:"read_percentage"`
-	TotalTimeSeconds  int64       `json:"total_time_seconds"`
-	LastRead          interface{} `json:"last_read"`
-	Percentage        float64     `json:"percentage"`
-	SecondsPerPercent int64       `json:"seconds_per_percent"`
-}
-
-func (q *Queries) GetDocumentWithStats(ctx context.Context, arg GetDocumentWithStatsParams) (GetDocumentWithStatsRow, error) {
-	row := q.db.QueryRowContext(ctx, getDocumentWithStats, arg.UserID, arg.DocumentID)
-	var i GetDocumentWithStatsRow
-	err := row.Scan(
-		&i.ID,
-		&i.Title,
-		&i.Author,
-		&i.Description,
-		&i.Isbn10,
-		&i.Isbn13,
-		&i.Filepath,
-		&i.Words,
-		&i.Wpm,
-		&i.ReadPercentage,
-		&i.TotalTimeSeconds,
-		&i.LastRead,
-		&i.Percentage,
-		&i.SecondsPerPercent,
-	)
-	return i, err
-}
-
 const getDocuments = `-- name: GetDocuments :many
 SELECT id, md5, basepath, filepath, coverfile, title, author, series, series_index, lang, description, words, gbid, olid, isbn10, isbn13, synced, deleted, updated_at, created_at FROM documents
 ORDER BY created_at DESC
@@ -719,37 +638,38 @@ SELECT
         WHEN (dus.percentage * 100.0) > 97.0 THEN 100.0
         ELSE dus.percentage * 100.0
     END AS REAL), 2) AS percentage,
-
-    CASE
+    CAST(CASE
         WHEN dus.total_time_seconds IS NULL THEN 0.0
         ELSE
-            ROUND(
-                CAST(dus.total_time_seconds AS REAL)
-                / (dus.read_percentage * 100.0)
-            )
-    END AS seconds_per_percent
+            CAST(dus.total_time_seconds AS REAL)
+            / (dus.read_percentage * 100.0)
+    END AS INTEGER) AS seconds_per_percent
 FROM documents AS docs
 LEFT JOIN users ON users.id = ?1
 LEFT JOIN
     document_user_statistics AS dus
     ON dus.document_id = docs.id AND dus.user_id = ?1
 WHERE
-    docs.deleted = false AND (
-        ?2 IS NULL OR (
-            docs.title LIKE ?2 OR
-            docs.author LIKE ?2
-        )
+    (docs.id = ?2 OR ?2 IS NULL)
+    AND (docs.deleted = ?3 OR ?3 IS NULL)
+    AND (
+        (
+            docs.title LIKE ?4 OR
+            docs.author LIKE ?4
+        ) OR ?4 IS NULL
     )
 ORDER BY dus.last_read DESC, docs.created_at DESC
-LIMIT ?4
-OFFSET ?3
+LIMIT ?6
+OFFSET ?5
 `
 
 type GetDocumentsWithStatsParams struct {
-	UserID string      `json:"user_id"`
-	Query  interface{} `json:"query"`
-	Offset int64       `json:"offset"`
-	Limit  int64       `json:"limit"`
+	UserID  string  `json:"user_id"`
+	ID      *string `json:"id"`
+	Deleted *bool   `json:"-"`
+	Query   *string `json:"query"`
+	Offset  int64   `json:"offset"`
+	Limit   int64   `json:"limit"`
 }
 
 type GetDocumentsWithStatsRow struct {
@@ -766,12 +686,14 @@ type GetDocumentsWithStatsRow struct {
 	TotalTimeSeconds  int64       `json:"total_time_seconds"`
 	LastRead          interface{} `json:"last_read"`
 	Percentage        float64     `json:"percentage"`
-	SecondsPerPercent interface{} `json:"seconds_per_percent"`
+	SecondsPerPercent int64       `json:"seconds_per_percent"`
 }
 
 func (q *Queries) GetDocumentsWithStats(ctx context.Context, arg GetDocumentsWithStatsParams) ([]GetDocumentsWithStatsRow, error) {
 	rows, err := q.db.QueryContext(ctx, getDocumentsWithStats,
 		arg.UserID,
+		arg.ID,
+		arg.Deleted,
 		arg.Query,
 		arg.Offset,
 		arg.Limit,
@@ -902,7 +824,7 @@ SELECT
     ROUND(CAST(progress.percentage AS REAL) * 100, 2) AS percentage,
     progress.document_id,
     progress.user_id,
-    CAST(STRFTIME('%Y-%m-%d %H:%M:%S', LOCAL_TIME(progress.created_at, users.timezone)) AS TEXT) AS created_at
+    LOCAL_TIME(progress.created_at, users.timezone) AS created_at
 FROM document_progress AS progress
 LEFT JOIN users ON progress.user_id = users.id
 LEFT JOIN devices ON progress.device_id = devices.id
@@ -929,13 +851,13 @@ type GetProgressParams struct {
 }
 
 type GetProgressRow struct {
-	Title      *string `json:"title"`
-	Author     *string `json:"author"`
-	DeviceName string  `json:"device_name"`
-	Percentage float64 `json:"percentage"`
-	DocumentID string  `json:"document_id"`
-	UserID     string  `json:"user_id"`
-	CreatedAt  string  `json:"created_at"`
+	Title      *string     `json:"title"`
+	Author     *string     `json:"author"`
+	DeviceName string      `json:"device_name"`
+	Percentage float64     `json:"percentage"`
+	DocumentID string      `json:"document_id"`
+	UserID     string      `json:"user_id"`
+	CreatedAt  interface{} `json:"created_at"`
 }
 
 func (q *Queries) GetProgress(ctx context.Context, arg GetProgressParams) ([]GetProgressRow, error) {
@@ -1078,7 +1000,7 @@ func (q *Queries) GetUserStatistics(ctx context.Context) ([]GetUserStatisticsRow
 }
 
 const getUserStreaks = `-- name: GetUserStreaks :many
-SELECT user_id, "window", max_streak, max_streak_start_date, max_streak_end_date, current_streak, current_streak_start_date, current_streak_end_date FROM user_streaks
+SELECT user_id, "window", max_streak, max_streak_start_date, max_streak_end_date, current_streak, current_streak_start_date, current_streak_end_date, last_timezone, last_seen, last_record, last_calculated FROM user_streaks
 WHERE user_id = ?1
 `
 
@@ -1100,6 +1022,10 @@ func (q *Queries) GetUserStreaks(ctx context.Context, userID string) ([]UserStre
 			&i.CurrentStreak,
 			&i.CurrentStreakStartDate,
 			&i.CurrentStreakEndDate,
+			&i.LastTimezone,
+			&i.LastSeen,
+			&i.LastRecord,
+			&i.LastCalculated,
 		); err != nil {
 			return nil, err
 		}

database/schema.sql

@@ -118,30 +118,13 @@ CREATE TABLE IF NOT EXISTS settings (
     created_at DATETIME NOT NULL DEFAULT (STRFTIME('%Y-%m-%dT%H:%M:%SZ', 'now'))
 );
 
----------------------------------------------------------------
------------------------ Temporary Tables ----------------------
----------------------------------------------------------------
-
--- Temporary User Streaks Table (Cached from View)
-CREATE TEMPORARY TABLE IF NOT EXISTS user_streaks (
-    user_id TEXT NOT NULL,
-    window TEXT NOT NULL,
-
-    max_streak INTEGER NOT NULL,
-    max_streak_start_date TEXT NOT NULL,
-    max_streak_end_date TEXT NOT NULL,
-
-    current_streak INTEGER NOT NULL,
-    current_streak_start_date TEXT NOT NULL,
-    current_streak_end_date TEXT NOT NULL
-);
-
--- Temporary Document User Statistics Table (Cached from View)
-CREATE TEMPORARY TABLE IF NOT EXISTS document_user_statistics (
+-- Document User Statistics Table
+CREATE TABLE IF NOT EXISTS document_user_statistics (
     document_id TEXT NOT NULL,
     user_id TEXT NOT NULL,
     percentage REAL NOT NULL,
-    last_read TEXT NOT NULL,
+    last_read DATETIME NOT NULL,
+    last_seen DATETIME NOT NULL,
     read_percentage REAL NOT NULL,
 
     total_time_seconds INTEGER NOT NULL,
@@ -163,21 +146,39 @@ CREATE TEMPORARY TABLE IF NOT EXISTS document_user_statistics (
     UNIQUE(document_id, user_id) ON CONFLICT REPLACE
 );
 
+-- User Streaks Table
+CREATE TABLE IF NOT EXISTS user_streaks (
+    user_id TEXT NOT NULL,
+    window TEXT NOT NULL,
+
+    max_streak INTEGER NOT NULL,
+    max_streak_start_date TEXT NOT NULL,
+    max_streak_end_date TEXT NOT NULL,
+
+    current_streak INTEGER NOT NULL,
+    current_streak_start_date TEXT NOT NULL,
+    current_streak_end_date TEXT NOT NULL,
+
+    last_timezone TEXT NOT NULL,
+    last_seen TEXT NOT NULL,
+    last_record TEXT NOT NULL,
+    last_calculated TEXT NOT NULL,
+
+    UNIQUE(user_id, window) ON CONFLICT REPLACE
+);
 
 ---------------------------------------------------------------
 --------------------------- Indexes ---------------------------
 ---------------------------------------------------------------
 
 CREATE INDEX IF NOT EXISTS activity_start_time ON activity (start_time);
+CREATE INDEX IF NOT EXISTS activity_created_at ON activity (created_at);
 CREATE INDEX IF NOT EXISTS activity_user_id ON activity (user_id);
 CREATE INDEX IF NOT EXISTS activity_user_id_document_id ON activity (
     user_id,
     document_id
 );
 
-DROP VIEW IF EXISTS view_user_streaks;
-DROP VIEW IF EXISTS view_document_user_statistics;
-
 ---------------------------------------------------------------
 --------------------------- Triggers --------------------------
 ---------------------------------------------------------------

database/user_streaks.sql

@@ -0,0 +1,154 @@
+WITH updated_users AS (
+    SELECT a.user_id
+    FROM activity AS a
+    LEFT JOIN users AS u ON u.id = a.user_id
+    LEFT JOIN user_streaks AS s ON a.user_id = s.user_id AND s.window = 'DAY'
+    WHERE
+        a.created_at > COALESCE(s.last_seen, '1970-01-01')
+        AND LOCAL_DATE(s.last_record, u.timezone) != LOCAL_DATE(a.start_time, u.timezone)
+    GROUP BY a.user_id
+),
+
+outdated_users AS (
+    SELECT
+        a.user_id,
+        u.timezone AS last_timezone,
+        MAX(a.created_at) AS last_seen,
+        MAX(a.start_time) AS last_record,
+        STRFTIME('%Y-%m-%dT%H:%M:%SZ', 'now') AS last_calculated
+    FROM activity AS a
+    LEFT JOIN users AS u ON u.id = a.user_id
+    LEFT JOIN user_streaks AS s ON a.user_id = s.user_id AND s.window = 'DAY'
+    GROUP BY a.user_id
+    HAVING
+        -- User Changed Timezones
+        s.last_timezone != u.timezone
+
+        -- Users Date Changed
+        OR LOCAL_DATE(COALESCE(s.last_calculated, '1970-01-01T00:00:00Z'), u.timezone) !=
+            LOCAL_DATE(STRFTIME('%Y-%m-%dT%H:%M:%SZ', 'now'), u.timezone)
+
+        -- User Added New Data
+        OR a.user_id IN updated_users
+),
+
+document_windows AS (
+    SELECT
+        activity.user_id,
+        users.timezone,
+        DATE(
+            LOCAL_DATE(activity.start_time, users.timezone),
+            'weekday 0', '-7 day'
+        ) AS weekly_read,
+        LOCAL_DATE(activity.start_time, users.timezone) AS daily_read
+    FROM activity
+    INNER JOIN outdated_users ON outdated_users.user_id = activity.user_id
+    LEFT JOIN users ON users.id = activity.user_id
+    GROUP BY activity.user_id, weekly_read, daily_read
+),
+
+weekly_partitions AS (
+    SELECT
+        user_id,
+        timezone,
+        'WEEK' AS "window",
+        weekly_read AS read_window,
+        ROW_NUMBER() OVER (
+            PARTITION BY user_id ORDER BY weekly_read DESC
+        ) AS seqnum
+    FROM document_windows
+    GROUP BY user_id, weekly_read
+),
+
+daily_partitions AS (
+    SELECT
+        user_id,
+        timezone,
+        'DAY' AS "window",
+        daily_read AS read_window,
+        ROW_NUMBER() OVER (
+            PARTITION BY user_id ORDER BY daily_read DESC
+        ) AS seqnum
+    FROM document_windows
+    GROUP BY user_id, daily_read
+),
+
+streaks AS (
+    SELECT
+        COUNT(*) AS streak,
+        MIN(read_window) AS start_date,
+        MAX(read_window) AS end_date,
+        window,
+        user_id,
+        timezone
+    FROM daily_partitions
+    GROUP BY
+        timezone,
+        user_id,
+        DATE(read_window, '+' || seqnum || ' day')
+
+    UNION ALL
+
+    SELECT
+        COUNT(*) AS streak,
+        MIN(read_window) AS start_date,
+        MAX(read_window) AS end_date,
+        window,
+        user_id,
+        timezone
+    FROM weekly_partitions
+    GROUP BY
+        timezone,
+        user_id,
+        DATE(read_window, '+' || (seqnum * 7) || ' day')
+),
+
+max_streak AS (
+    SELECT
+        MAX(streak) AS max_streak,
+        start_date AS max_streak_start_date,
+        end_date AS max_streak_end_date,
+        window,
+        user_id
+    FROM streaks
+    GROUP BY user_id, window
+),
+
+current_streak AS (
+    SELECT
+        streak AS current_streak,
+        start_date AS current_streak_start_date,
+        end_date AS current_streak_end_date,
+        window,
+        user_id
+    FROM streaks
+    WHERE CASE
+      WHEN window = "WEEK" THEN
+          DATE(LOCAL_DATE(STRFTIME('%Y-%m-%dT%H:%M:%SZ', 'now'), timezone), 'weekday 0', '-14 day') = current_streak_end_date
+          OR DATE(LOCAL_DATE(STRFTIME('%Y-%m-%dT%H:%M:%SZ', 'now'), timezone), 'weekday 0', '-7 day') = current_streak_end_date
+      WHEN window = "DAY" THEN
+          DATE(LOCAL_DATE(STRFTIME('%Y-%m-%dT%H:%M:%SZ', 'now'), timezone), '-1 day') = current_streak_end_date
+          OR DATE(LOCAL_DATE(STRFTIME('%Y-%m-%dT%H:%M:%SZ', 'now'), timezone)) = current_streak_end_date
+    END
+    GROUP BY user_id, window
+)
+
+INSERT INTO user_streaks
+SELECT
+    max_streak.user_id,
+    max_streak.window,
+    IFNULL(max_streak, 0) AS max_streak,
+    IFNULL(max_streak_start_date, "N/A") AS max_streak_start_date,
+    IFNULL(max_streak_end_date, "N/A") AS max_streak_end_date,
+    IFNULL(current_streak.current_streak, 0) AS current_streak,
+    IFNULL(current_streak.current_streak_start_date, "N/A") AS current_streak_start_date,
+    IFNULL(current_streak.current_streak_end_date, "N/A") AS current_streak_end_date,
+    outdated_users.last_timezone AS last_timezone,
+    outdated_users.last_seen AS last_seen,
+    outdated_users.last_record AS last_record,
+    outdated_users.last_calculated AS last_calculated
+FROM max_streak
+JOIN outdated_users ON max_streak.user_id = outdated_users.user_id
+LEFT JOIN current_streak ON
+    current_streak.user_id = max_streak.user_id
+    AND current_streak.window = max_streak.window;

database/users_test.go

@@ -1,6 +1,7 @@
 package database
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
 	"testing"
@@ -36,7 +37,7 @@ func (suite *UsersTestSuite) SetupTest() {
 	// Create User
 	rawAuthHash, _ := utils.GenerateToken(64)
 	authHash := fmt.Sprintf("%x", rawAuthHash)
-	_, err := suite.dbm.Queries.CreateUser(suite.dbm.Ctx, CreateUserParams{
+	_, err := suite.dbm.Queries.CreateUser(context.Background(), CreateUserParams{
 		ID:       testUserID,
 		Pass:     &testUserPass,
 		AuthHash: &authHash,
@@ -44,7 +45,7 @@ func (suite *UsersTestSuite) SetupTest() {
 	suite.NoError(err)
 
 	// Create Document
-	_, err = suite.dbm.Queries.UpsertDocument(suite.dbm.Ctx, UpsertDocumentParams{
+	_, err = suite.dbm.Queries.UpsertDocument(context.Background(), UpsertDocumentParams{
 		ID:     documentID,
 		Title:  &documentTitle,
 		Author: &documentAuthor,
@@ -53,7 +54,7 @@ func (suite *UsersTestSuite) SetupTest() {
 	suite.NoError(err)
 
 	// Create Device
-	_, err = suite.dbm.Queries.UpsertDevice(suite.dbm.Ctx, UpsertDeviceParams{
+	_, err = suite.dbm.Queries.UpsertDevice(context.Background(), UpsertDeviceParams{
 		ID:         deviceID,
 		UserID:     testUserID,
 		DeviceName: deviceName,
@@ -62,7 +63,7 @@ func (suite *UsersTestSuite) SetupTest() {
 }
 
 func (suite *UsersTestSuite) TestGetUser() {
-	user, err := suite.dbm.Queries.GetUser(suite.dbm.Ctx, testUserID)
+	user, err := suite.dbm.Queries.GetUser(context.Background(), testUserID)
 	suite.Nil(err, "should have nil err")
 	suite.Equal(testUserPass, *user.Pass)
 }
@@ -76,7 +77,7 @@ func (suite *UsersTestSuite) TestCreateUser() {
 	suite.Nil(err, "should have nil err")
 
 	authHash := fmt.Sprintf("%x", rawAuthHash)
-	changed, err := suite.dbm.Queries.CreateUser(suite.dbm.Ctx, CreateUserParams{
+	changed, err := suite.dbm.Queries.CreateUser(context.Background(), CreateUserParams{
 		ID:       testUser,
 		Pass:     &testPass,
 		AuthHash: &authHash,
@@ -85,29 +86,29 @@ func (suite *UsersTestSuite) TestCreateUser() {
 	suite.Nil(err, "should have nil err")
 	suite.Equal(int64(1), changed)
 
-	user, err := suite.dbm.Queries.GetUser(suite.dbm.Ctx, testUser)
+	user, err := suite.dbm.Queries.GetUser(context.Background(), testUser)
 	suite.Nil(err, "should have nil err")
 	suite.Equal(testPass, *user.Pass)
 }
 
 func (suite *UsersTestSuite) TestDeleteUser() {
-	changed, err := suite.dbm.Queries.DeleteUser(suite.dbm.Ctx, testUserID)
+	changed, err := suite.dbm.Queries.DeleteUser(context.Background(), testUserID)
 	suite.Nil(err, "should have nil err")
 	suite.Equal(int64(1), changed, "should have one changed row")
 
-	_, err = suite.dbm.Queries.GetUser(suite.dbm.Ctx, testUserID)
+	_, err = suite.dbm.Queries.GetUser(context.Background(), testUserID)
 	suite.ErrorIs(err, sql.ErrNoRows, "should have no rows error")
 }
 
 func (suite *UsersTestSuite) TestGetUsers() {
-	users, err := suite.dbm.Queries.GetUsers(suite.dbm.Ctx)
+	users, err := suite.dbm.Queries.GetUsers(context.Background())
 	suite.Nil(err, "should have nil err")
 	suite.Len(users, 1, "should have single user")
 }
 
 func (suite *UsersTestSuite) TestUpdateUser() {
 	newPassword := "newPass123"
-	user, err := suite.dbm.Queries.UpdateUser(suite.dbm.Ctx, UpdateUserParams{
+	user, err := suite.dbm.Queries.UpdateUser(context.Background(), UpdateUserParams{
 		UserID:   testUserID,
 		Password: &newPassword,
 	})
@@ -116,11 +117,11 @@ func (suite *UsersTestSuite) TestUpdateUser() {
 }
 
 func (suite *UsersTestSuite) TestGetUserStatistics() {
-	err := suite.dbm.CacheTempTables()
+	err := suite.dbm.CacheTempTables(context.Background())
 	suite.NoError(err)
 
 	// Ensure Zero Items
-	userStats, err := suite.dbm.Queries.GetUserStatistics(suite.dbm.Ctx)
+	userStats, err := suite.dbm.Queries.GetUserStatistics(context.Background())
 	suite.Nil(err, "should have nil err")
 	suite.Empty(userStats, "should be empty")
 
@@ -133,7 +134,7 @@ func (suite *UsersTestSuite) TestGetUserStatistics() {
 		counter += 1
 
 		// Add Item
-		activity, err := suite.dbm.Queries.AddActivity(suite.dbm.Ctx, AddActivityParams{
+		activity, err := suite.dbm.Queries.AddActivity(context.Background(), AddActivityParams{
 			DocumentID:      documentID,
 			DeviceID:        deviceID,
 			UserID:          testUserID,
@@ -147,21 +148,21 @@ func (suite *UsersTestSuite) TestGetUserStatistics() {
 		suite.Equal(counter, activity.ID, fmt.Sprintf("[%d] should have correct id for add activity", counter))
 	}
 
-	err = suite.dbm.CacheTempTables()
+	err = suite.dbm.CacheTempTables(context.Background())
 	suite.NoError(err)
 
 	// Ensure One Item
-	userStats, err = suite.dbm.Queries.GetUserStatistics(suite.dbm.Ctx)
+	userStats, err = suite.dbm.Queries.GetUserStatistics(context.Background())
 	suite.Nil(err, "should have nil err")
 	suite.Len(userStats, 1, "should have length of one")
 }
 
 func (suite *UsersTestSuite) TestGetUsersStreaks() {
-	err := suite.dbm.CacheTempTables()
+	err := suite.dbm.CacheTempTables(context.Background())
 	suite.NoError(err)
 
 	// Ensure Zero Items
-	userStats, err := suite.dbm.Queries.GetUserStreaks(suite.dbm.Ctx, testUserID)
+	userStats, err := suite.dbm.Queries.GetUserStreaks(context.Background(), testUserID)
 	suite.Nil(err, "should have nil err")
 	suite.Empty(userStats, "should be empty")
 
@@ -174,7 +175,7 @@ func (suite *UsersTestSuite) TestGetUsersStreaks() {
 		counter += 1
 
 		// Add Item
-		activity, err := suite.dbm.Queries.AddActivity(suite.dbm.Ctx, AddActivityParams{
+		activity, err := suite.dbm.Queries.AddActivity(context.Background(), AddActivityParams{
 			DocumentID:      documentID,
 			DeviceID:        deviceID,
 			UserID:          testUserID,
@@ -188,11 +189,11 @@ func (suite *UsersTestSuite) TestGetUsersStreaks() {
 		suite.Equal(counter, activity.ID, fmt.Sprintf("[%d] should have correct id for add activity", counter))
 	}
 
-	err = suite.dbm.CacheTempTables()
+	err = suite.dbm.CacheTempTables(context.Background())
 	suite.NoError(err)
 
 	// Ensure Two Item
-	userStats, err = suite.dbm.Queries.GetUserStreaks(suite.dbm.Ctx, testUserID)
+	userStats, err = suite.dbm.Queries.GetUserStreaks(context.Background(), testUserID)
 	suite.Nil(err, "should have nil err")
 	suite.Len(userStats, 2, "should have length of two")
 

database/views.sql

@@ -1,296 +0,0 @@
----------------------------------------------------------------
----------------------------- Views ----------------------------
----------------------------------------------------------------
-
---------------------------------
---------- User Streaks ---------
---------------------------------
-
-CREATE VIEW view_user_streaks AS
-
-WITH document_windows AS (
-    SELECT
-        activity.user_id,
-        users.timezone,
-        DATE(
-            LOCAL_TIME(activity.start_time, users.timezone),
-            'weekday 0', '-7 day'
-        ) AS weekly_read,
-        DATE(LOCAL_TIME(activity.start_time, users.timezone)) AS daily_read
-    FROM activity
-    LEFT JOIN users ON users.id = activity.user_id
-    GROUP BY activity.user_id, weekly_read, daily_read
-),
-weekly_partitions AS (
-    SELECT
-        user_id,
-        timezone,
-        'WEEK' AS "window",
-        weekly_read AS read_window,
-        row_number() OVER (
-            PARTITION BY user_id ORDER BY weekly_read DESC
-        ) AS seqnum
-    FROM document_windows
-    GROUP BY user_id, weekly_read
-),
-daily_partitions AS (
-    SELECT
-        user_id,
-        timezone,
-        'DAY' AS "window",
-        daily_read AS read_window,
-        row_number() OVER (
-            PARTITION BY user_id ORDER BY daily_read DESC
-        ) AS seqnum
-    FROM document_windows
-    GROUP BY user_id, daily_read
-),
-streaks AS (
-    SELECT
-        COUNT(*) AS streak,
-        MIN(read_window) AS start_date,
-        MAX(read_window) AS end_date,
-        window,
-        user_id,
-        timezone
-    FROM daily_partitions
-    GROUP BY
-        timezone,
-        user_id,
-        DATE(read_window, '+' || seqnum || ' day')
-
-    UNION ALL
-
-    SELECT
-        COUNT(*) AS streak,
-        MIN(read_window) AS start_date,
-        MAX(read_window) AS end_date,
-        window,
-        user_id,
-        timezone
-    FROM weekly_partitions
-    GROUP BY
-        timezone,
-        user_id,
-        DATE(read_window, '+' || (seqnum * 7) || ' day')
-),
-max_streak AS (
-    SELECT
-        MAX(streak) AS max_streak,
-        start_date AS max_streak_start_date,
-        end_date AS max_streak_end_date,
-        window,
-        user_id
-    FROM streaks
-    GROUP BY user_id, window
-),
-current_streak AS (
-    SELECT
-        streak AS current_streak,
-        start_date AS current_streak_start_date,
-        end_date AS current_streak_end_date,
-        window,
-        user_id
-    FROM streaks
-    WHERE CASE
-      WHEN window = "WEEK" THEN
-          DATE(LOCAL_TIME(STRFTIME('%Y-%m-%dT%H:%M:%SZ', 'now'), timezone), 'weekday 0', '-14 day') = current_streak_end_date
-          OR DATE(LOCAL_TIME(STRFTIME('%Y-%m-%dT%H:%M:%SZ', 'now'), timezone), 'weekday 0', '-7 day') = current_streak_end_date
-      WHEN window = "DAY" THEN
-          DATE(LOCAL_TIME(STRFTIME('%Y-%m-%dT%H:%M:%SZ', 'now'), timezone), '-1 day') = current_streak_end_date
-          OR DATE(LOCAL_TIME(STRFTIME('%Y-%m-%dT%H:%M:%SZ', 'now'), timezone)) = current_streak_end_date
-    END
-    GROUP BY user_id, window
-)
-SELECT
-    max_streak.user_id,
-    max_streak.window,
-    IFNULL(max_streak, 0) AS max_streak,
-    IFNULL(max_streak_start_date, "N/A") AS max_streak_start_date,
-    IFNULL(max_streak_end_date, "N/A") AS max_streak_end_date,
-    IFNULL(current_streak, 0) AS current_streak,
-    IFNULL(current_streak_start_date, "N/A") AS current_streak_start_date,
-    IFNULL(current_streak_end_date, "N/A") AS current_streak_end_date
-FROM max_streak
-LEFT JOIN current_streak ON
-    current_streak.user_id = max_streak.user_id
-    AND current_streak.window = max_streak.window;
-
---------------------------------
-------- Document Stats ---------
---------------------------------
-
-CREATE VIEW view_document_user_statistics AS
-
-WITH intermediate_ga AS (
-    SELECT
-        ga1.id AS row_id,
-        ga1.user_id,
-        ga1.document_id,
-        ga1.duration,
-        ga1.start_time,
-        ga1.start_percentage,
-        ga1.end_percentage,
-
-        -- Find Overlapping Events (Assign Unique ID)
-        (
-            SELECT MIN(id)
-            FROM activity AS ga2
-            WHERE
-                ga1.document_id = ga2.document_id
-                AND ga1.user_id = ga2.user_id
-                AND ga1.start_percentage <= ga2.end_percentage
-                AND ga1.end_percentage >= ga2.start_percentage
-        ) AS group_leader
-    FROM activity AS ga1
-),
-
-grouped_activity AS (
-    SELECT
-        user_id,
-        document_id,
-        MAX(start_time) AS start_time,
-        MIN(start_percentage) AS start_percentage,
-        MAX(end_percentage) AS end_percentage,
-        MAX(end_percentage) - MIN(start_percentage) AS read_percentage,
-        SUM(duration) AS duration
-    FROM intermediate_ga
-    GROUP BY group_leader
-),
-
-current_progress AS (
-    SELECT
-        user_id,
-        document_id,
-        COALESCE((
-            SELECT percentage
-            FROM document_progress AS dp
-            WHERE
-                dp.user_id = iga.user_id
-                AND dp.document_id = iga.document_id
-            ORDER BY created_at DESC
-            LIMIT 1
-        ), end_percentage) AS percentage
-    FROM intermediate_ga AS iga
-    GROUP BY user_id, document_id
-    HAVING MAX(start_time)
-)
-
-SELECT
-    ga.document_id,
-    ga.user_id,
-    cp.percentage,
-    MAX(start_time) AS last_read,
-    SUM(read_percentage) AS read_percentage,
-
-    -- All Time WPM
-    SUM(duration) AS total_time_seconds,
-    (CAST(COALESCE(d.words, 0.0) AS REAL) * SUM(read_percentage))
-        AS total_words_read,
-    (CAST(COALESCE(d.words, 0.0) AS REAL) * SUM(read_percentage))
-    / (SUM(duration) / 60.0) AS total_wpm,
-
-    -- Yearly WPM
-    SUM(CASE WHEN start_time >= DATE('now', '-1 year') THEN duration ELSE 0 END)
-        AS yearly_time_seconds,
-    (
-        CAST(COALESCE(d.words, 0.0) AS REAL)
-        * SUM(
-            CASE
-                WHEN start_time >= DATE('now', '-1 year') THEN read_percentage
-                ELSE 0
-            END
-        )
-    )
-        AS yearly_words_read,
-    COALESCE((
-        CAST(COALESCE(d.words, 0.0) AS REAL)
-        * SUM(
-            CASE
-                WHEN start_time >= DATE('now', '-1 year') THEN read_percentage
-            END
-        )
-    )
-    / (
-        SUM(
-            CASE
-                WHEN start_time >= DATE('now', '-1 year') THEN duration
-            END
-        )
-        / 60.0
-    ), 0.0)
-        AS yearly_wpm,
-
-    -- Monthly WPM
-    SUM(
-        CASE WHEN start_time >= DATE('now', '-1 month') THEN duration ELSE 0 END
-    )
-        AS monthly_time_seconds,
-    (
-        CAST(COALESCE(d.words, 0.0) AS REAL)
-        * SUM(
-            CASE
-                WHEN start_time >= DATE('now', '-1 month') THEN read_percentage
-                ELSE 0
-            END
-        )
-    )
-        AS monthly_words_read,
-    COALESCE((
-        CAST(COALESCE(d.words, 0.0) AS REAL)
-        * SUM(
-            CASE
-                WHEN start_time >= DATE('now', '-1 month') THEN read_percentage
-            END
-        )
-    )
-    / (
-        SUM(
-            CASE
-                WHEN start_time >= DATE('now', '-1 month') THEN duration
-            END
-        )
-        / 60.0
-    ), 0.0)
-        AS monthly_wpm,
-
-    -- Weekly WPM
-    SUM(CASE WHEN start_time >= DATE('now', '-7 days') THEN duration ELSE 0 END)
-        AS weekly_time_seconds,
-    (
-        CAST(COALESCE(d.words, 0.0) AS REAL)
-        * SUM(
-            CASE
-                WHEN start_time >= DATE('now', '-7 days') THEN read_percentage
-                ELSE 0
-            END
-        )
-    )
-        AS weekly_words_read,
-    COALESCE((
-        CAST(COALESCE(d.words, 0.0) AS REAL)
-        * SUM(
-            CASE
-                WHEN start_time >= DATE('now', '-7 days') THEN read_percentage
-            END
-        )
-    )
-    / (
-        SUM(
-            CASE
-                WHEN start_time >= DATE('now', '-7 days') THEN duration
-            END
-        )
-        / 60.0
-    ), 0.0)
-        AS weekly_wpm
-
-FROM grouped_activity AS ga
-INNER JOIN
-    current_progress AS cp
-    ON ga.user_id = cp.user_id AND ga.document_id = cp.document_id
-INNER JOIN
-    documents AS d
-    ON ga.document_id = d.id
-GROUP BY ga.document_id, ga.user_id
-ORDER BY total_wpm DESC;

flake.lock

@@ -0,0 +1,61 @@
+{
+  "nodes": {
+    "flake-utils": {
+      "inputs": {
+        "systems": "systems"
+      },
+      "locked": {
+        "lastModified": 1731533236,
+        "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1764522689,
+        "narHash": "sha256-SqUuBFjhl/kpDiVaKLQBoD8TLD+/cTUzzgVFoaHrkqY=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "8bb5646e0bed5dbd3ab08c7a7cc15b75ab4e1d0f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-25.11",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "flake-utils": "flake-utils",
+        "nixpkgs": "nixpkgs"
+      }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

flake.nix

@@ -0,0 +1,36 @@
+{
+  description = "Development Environment";
+
+  inputs = {
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.11";
+    flake-utils.url = "github:numtide/flake-utils";
+  };
+
+  outputs =
+    { self
+    , nixpkgs
+    , flake-utils
+    ,
+    }:
+    flake-utils.lib.eachDefaultSystem (
+      system:
+      let
+        pkgs = nixpkgs.legacyPackages.${system};
+      in
+      {
+        devShells.default = pkgs.mkShell {
+          packages = with pkgs; [
+            go
+            gopls
+            golangci-lint
+            nodejs
+            tailwindcss
+            python311Packages.grip
+          ];
+          shellHook = ''
+            export PATH=$PATH:~/go/bin
+          '';
+        };
+      }
+    );
+}

go.mod

@@ -1,81 +1,85 @@
 module reichard.io/antholume
 
-go 1.21
+go 1.24
 
 require (
-	github.com/PuerkitoBio/goquery v1.8.1
+	github.com/PuerkitoBio/goquery v1.10.3
 	github.com/alexedwards/argon2id v1.0.0
-	github.com/gabriel-vasile/mimetype v1.4.3
-	github.com/gin-contrib/multitemplate v0.0.0-20231230012943-32b233489a81
-	github.com/gin-contrib/sessions v0.0.5
-	github.com/gin-gonic/gin v1.9.1
-	github.com/itchyny/gojq v0.12.14
-	github.com/microcosm-cc/bluemonday v1.0.26
-	github.com/pressly/goose/v3 v3.17.0
+	github.com/gabriel-vasile/mimetype v1.4.9
+	github.com/gin-contrib/multitemplate v1.1.1
+	github.com/gin-contrib/sessions v1.0.4
+	github.com/gin-gonic/gin v1.10.1
+	github.com/itchyny/gojq v0.12.17
+	github.com/jarcoal/httpmock v1.3.1
+	github.com/microcosm-cc/bluemonday v1.0.27
+	github.com/pkg/errors v0.9.1
+	github.com/pressly/goose/v3 v3.24.3
 	github.com/sirupsen/logrus v1.9.3
-	github.com/taylorskalyo/goreader v0.0.0-20230626212555-e7f5644f8115
-	github.com/urfave/cli/v2 v2.27.1
-	golang.org/x/exp v0.0.0-20240119083558-1b970713d09a
+	github.com/stretchr/testify v1.10.0
+	github.com/taylorskalyo/goreader v1.0.1
+	github.com/urfave/cli/v2 v2.27.7
+	golang.org/x/exp v0.0.0-20250718183923-645b1fa84792
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1
-	modernc.org/sqlite v1.28.0
+	modernc.org/sqlite v1.38.2
 )
 
 require (
-	github.com/andybalholm/cascadia v1.3.2 // indirect
+	github.com/andybalholm/cascadia v1.3.3 // indirect
 	github.com/aymerick/douceur v0.2.0 // indirect
-	github.com/bytedance/sonic v1.10.2 // indirect
+	github.com/bytedance/sonic v1.14.0 // indirect
+	github.com/bytedance/sonic/loader v0.3.0 // indirect
 	github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect
 	github.com/chenzhuoyu/iasm v0.9.1 // indirect
-	github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect
+	github.com/cloudwego/base64x v0.1.6 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
-	github.com/gin-contrib/sse v0.1.0 // indirect
+	github.com/gin-contrib/sse v1.1.0 // indirect
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
-	github.com/go-playground/validator/v10 v10.17.0 // indirect
-	github.com/goccy/go-json v0.10.2 // indirect
+	github.com/go-playground/validator/v10 v10.27.0 // indirect
+	github.com/goccy/go-json v0.10.5 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/gorilla/context v1.1.2 // indirect
 	github.com/gorilla/css v1.0.1 // indirect
 	github.com/gorilla/securecookie v1.1.2 // indirect
-	github.com/gorilla/sessions v1.2.2 // indirect
-	github.com/itchyny/timefmt-go v0.1.5 // indirect
-	github.com/jarcoal/httpmock v1.3.1 // indirect
+	github.com/gorilla/sessions v1.4.0 // indirect
+	github.com/itchyny/timefmt-go v0.1.6 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.6 // indirect
-	github.com/leodido/go-urn v1.2.4 // indirect
+	github.com/klauspost/cpuid/v2 v2.3.0 // indirect
+	github.com/leodido/go-urn v1.4.0 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mfridman/interpolate v0.0.2 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
-	github.com/pelletier/go-toml/v2 v2.1.1 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
+	github.com/ncruces/go-strftime v0.1.9 // indirect
+	github.com/pelletier/go-toml/v2 v2.2.4 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
-	github.com/sethvargo/go-retry v0.2.4 // indirect
-	github.com/stretchr/testify v1.8.4 // indirect
+	github.com/sethvargo/go-retry v0.3.0 // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
-	github.com/ugorji/go/codec v1.2.12 // indirect
-	github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e // indirect
+	github.com/ugorji/go/codec v1.3.0 // indirect
+	github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	golang.org/x/arch v0.7.0 // indirect
-	golang.org/x/crypto v0.18.0 // indirect
-	golang.org/x/mod v0.14.0 // indirect
-	golang.org/x/net v0.20.0 // indirect
-	golang.org/x/sync v0.6.0 // indirect
-	golang.org/x/sys v0.16.0 // indirect
-	golang.org/x/text v0.14.0 // indirect
-	golang.org/x/tools v0.17.0 // indirect
-	google.golang.org/protobuf v1.32.0 // indirect
+	golang.org/x/arch v0.20.0 // indirect
+	golang.org/x/crypto v0.41.0 // indirect
+	golang.org/x/mod v0.27.0 // indirect
+	golang.org/x/net v0.43.0 // indirect
+	golang.org/x/sync v0.16.0 // indirect
+	golang.org/x/sys v0.35.0 // indirect
+	golang.org/x/text v0.28.0 // indirect
+	golang.org/x/tools v0.36.0 // indirect
+	google.golang.org/protobuf v1.36.7 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	lukechampine.com/uint128 v1.3.0 // indirect
 	modernc.org/cc/v3 v3.41.0 // indirect
-	modernc.org/ccgo/v3 v3.16.15 // indirect
-	modernc.org/libc v1.40.7 // indirect
-	modernc.org/mathutil v1.6.0 // indirect
-	modernc.org/memory v1.7.2 // indirect
-	modernc.org/opt v0.1.3 // indirect
-	modernc.org/strutil v1.2.0 // indirect
+	modernc.org/ccgo/v3 v3.17.0 // indirect
+	modernc.org/libc v1.66.6 // indirect
+	modernc.org/mathutil v1.7.1 // indirect
+	modernc.org/memory v1.11.0 // indirect
+	modernc.org/opt v0.1.4 // indirect
+	modernc.org/strutil v1.2.1 // indirect
 	modernc.org/token v1.1.0 // indirect
 )

go.sum

@@ -2,27 +2,38 @@ github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25
 github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
 github.com/ClickHouse/ch-go v0.58.2 h1:jSm2szHbT9MCAB1rJ3WuCJqmGLi5UTjlNu+f530UTS0=
 github.com/ClickHouse/ch-go v0.58.2/go.mod h1:Ap/0bEmiLa14gYjCiRkYGbXvbe8vwdrfTYWhsuQ99aw=
+github.com/ClickHouse/ch-go v0.65.1 h1:SLuxmLl5Mjj44/XbINsK2HFvzqup0s6rwKLFH347ZhU=
 github.com/ClickHouse/clickhouse-go/v2 v2.16.0 h1:rhMfnPewXPnY4Q4lQRGdYuTLRBRKJEIEYHtbUMrzmvI=
 github.com/ClickHouse/clickhouse-go/v2 v2.16.0/go.mod h1:J7SPfIxwR+x4mQ+o8MLSe0oY50NNntEqCIjFe/T1VPM=
+github.com/ClickHouse/clickhouse-go/v2 v2.34.0 h1:Y4rqkdrRHgExvC4o/NTbLdY5LFQ3LHS77/RNFxFX3Co=
 github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
 github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
 github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw=
 github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk=
 github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM=
 github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ=
+github.com/PuerkitoBio/goquery v1.10.3 h1:pFYcNSqHxBD06Fpj/KsbStFRsgRATgnf3LeXiUkhzPo=
+github.com/PuerkitoBio/goquery v1.10.3/go.mod h1:tMUX0zDMHXYlAQk6p35XxQMqMweEKB7iK7iLNd4RH4Y=
 github.com/alexedwards/argon2id v1.0.0 h1:wJzDx66hqWX7siL/SRUmgz3F8YMrd/nfX/xHHcQQP0w=
 github.com/alexedwards/argon2id v1.0.0/go.mod h1:tYKkqIjzXvZdzPvADMWOEZ+l6+BD6CtBXMj5fnJppiw=
 github.com/andybalholm/brotli v1.0.6 h1:Yf9fFpf49Zrxb9NlQaluyE92/+X7UVHlhMNJN2sxfOI=
 github.com/andybalholm/brotli v1.0.6/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
+github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
 github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
 github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
 github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
+github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
+github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
 github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
 github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
 github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM=
 github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM=
 github.com/bytedance/sonic v1.10.2 h1:GQebETVBxYB7JGWJtLBi07OVzWwt+8dWA00gEVW2ZFE=
 github.com/bytedance/sonic v1.10.2/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4=
+github.com/bytedance/sonic v1.14.0 h1:/OfKt8HFw0kh2rj8N0F6C/qPGRESq0BbaNZgcNXXzQQ=
+github.com/bytedance/sonic v1.14.0/go.mod h1:WoEbx8WTcFJfzCe0hbmyTGrfjt8PzNEBdxlNUO24NhA=
+github.com/bytedance/sonic/loader v0.3.0 h1:dskwH8edlzNMctoruo8FPTJDF3vLtDT0sXZwvZJyqeA=
+github.com/bytedance/sonic/loader v0.3.0/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI=
 github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
 github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY=
@@ -32,10 +43,15 @@ github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d/go.mod h1:8EPpV
 github.com/chenzhuoyu/iasm v0.9.0/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog=
 github.com/chenzhuoyu/iasm v0.9.1 h1:tUHQJXo3NhBqw6s33wkGn9SP3bvrWLdlVIJ3hQBL7P0=
 github.com/chenzhuoyu/iasm v0.9.1/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog=
+github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M=
+github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU=
+github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY=
 github.com/containerd/continuity v0.4.3 h1:6HVkalIp+2u1ZLH1J/pYX2oBVXlJZvh1X1A7bEZ9Su8=
 github.com/containerd/continuity v0.4.3/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ=
 github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM=
 github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo=
+github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -52,23 +68,36 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/elastic/go-sysinfo v1.11.2 h1:mcm4OSYVMyws6+n2HIVMGkln5HOpo5Ie1ZmbbNn0jg4=
 github.com/elastic/go-sysinfo v1.11.2/go.mod h1:GKqR8bbMK/1ITnez9NIsIfXQr25aLhRJa7AfT8HpBFQ=
+github.com/elastic/go-sysinfo v1.15.3 h1:W+RnmhKFkqPTCRoFq2VCTmsT4p/fwpo+3gKNQsn1XU0=
 github.com/elastic/go-windows v1.0.1 h1:AlYZOldA+UJ0/2nBuqWdo90GFCgG9xuyw9SYzGUtJm0=
 github.com/elastic/go-windows v1.0.1/go.mod h1:FoVvqWSun28vaDQPbj2Elfc0JahhPB7WQEGa3c814Ss=
+github.com/elastic/go-windows v1.0.2 h1:yoLLsAsV5cfg9FLhZ9EXZ2n2sQFKeDYrHenkcivY4vI=
 github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
 github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
+github.com/gabriel-vasile/mimetype v1.4.9 h1:5k+WDwEsD9eTLL8Tz3L0VnmVh9QxGjRmjBvAG7U/oYY=
+github.com/gabriel-vasile/mimetype v1.4.9/go.mod h1:WnSQhFKJuBlRyLiKohA/2DtIlPFAbguNaG7QCHcyGok=
 github.com/gin-contrib/multitemplate v0.0.0-20231230012943-32b233489a81 h1:hQ/WeoPMTbN8NHk5i96dWy3D4uF7yCU+kORyWG+P4oU=
 github.com/gin-contrib/multitemplate v0.0.0-20231230012943-32b233489a81/go.mod h1:XLLtIXoP9+9zGcEDc7gAGV3AksGPO+vzv4kXHMJSdU0=
+github.com/gin-contrib/multitemplate v1.1.1 h1:uzhT/ZWS9nBd1h6P+AaxWaVSVAJRAcKH4yafrBU8sPc=
+github.com/gin-contrib/multitemplate v1.1.1/go.mod h1:1Sa4984P8+x87U0cg5yWxK4jpbK1cXMYegUCZK6XT/M=
 github.com/gin-contrib/sessions v0.0.5 h1:CATtfHmLMQrMNpJRgzjWXD7worTh7g7ritsQfmF+0jE=
 github.com/gin-contrib/sessions v0.0.5/go.mod h1:vYAuaUPqie3WUSsft6HUlCjlwwoJQs97miaG2+7neKY=
+github.com/gin-contrib/sessions v1.0.4 h1:ha6CNdpYiTOK/hTp05miJLbpTSNfOnFg5Jm2kbcqy8U=
+github.com/gin-contrib/sessions v1.0.4/go.mod h1:ccmkrb2z6iU2osiAHZG3x3J4suJK+OU27oqzlWOqQgs=
 github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
 github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
+github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w=
+github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM=
 github.com/gin-gonic/gin v1.8.1/go.mod h1:ji8BvRH1azfM+SYow9zQ6SZMvR8qOMZHmsCuWR9tTTk=
 github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg=
 github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU=
+github.com/gin-gonic/gin v1.10.1 h1:T0ujvqyCSqRopADpgPgiTT63DUQVSfojyME59Ei63pQ=
+github.com/gin-gonic/gin v1.10.1/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y=
 github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw=
 github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw=
 github.com/go-faster/errors v0.6.1 h1:nNIPOBkprlKzkThvS/0YaX8Zs9KewLCOSFQS5BU06FI=
 github.com/go-faster/errors v0.6.1/go.mod h1:5MGV2/2T9yvlrbhe9pD9LO5Z/2zCSq2T8j+Jpi2LAyY=
+github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg=
 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
 github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
@@ -81,26 +110,36 @@ github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91
 github.com/go-playground/validator/v10 v10.10.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos=
 github.com/go-playground/validator/v10 v10.17.0 h1:SmVVlfAOtlZncTxRuinDPomC2DkXJ4E5T9gDA0AIH74=
 github.com/go-playground/validator/v10 v10.17.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
+github.com/go-playground/validator/v10 v10.27.0 h1:w8+XrWVMhGkxOaaowyKH35gFydVHOvC0/uWoy2Fzwn4=
+github.com/go-playground/validator/v10 v10.27.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo=
 github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI=
 github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
+github.com/go-sql-driver/mysql v1.9.2 h1:4cNKDYQ1I84SXslGddlsrMhc8k4LeDVj6Ad6WRjiHuU=
 github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
 github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
+github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
+github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg=
 github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
+github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI=
+github.com/golang/protobuf v1.5.0 h1:LUVKkCeviFUMKqHa4tXIIij/lbhnMbP7Fn5wKdKkRh4=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
 github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ=
 github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo=
+github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs=
 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
@@ -113,35 +152,48 @@ github.com/gorilla/securecookie v1.1.2 h1:YCIWL56dvtr73r6715mJs5ZvhtnY73hBvEF8kX
 github.com/gorilla/securecookie v1.1.2/go.mod h1:NfCASbcHqRSY+3a8tlWJwsQap2VX5pwzwo4h3eOamfo=
 github.com/gorilla/sessions v1.2.2 h1:lqzMYz6bOfvn2WriPUjNByzeXIlVzURcPmgMczkmTjY=
 github.com/gorilla/sessions v1.2.2/go.mod h1:ePLdVu+jbEgHH+KWw8I1z2wqd0BAdAQh/8LRvBeoNcQ=
+github.com/gorilla/sessions v1.4.0 h1:kpIYOp/oi6MG/p5PgxApU8srsSw9tuFbt46Lt7auzqQ=
+github.com/gorilla/sessions v1.4.0/go.mod h1:FLWm50oby91+hl7p/wRxDth9bWSuk0qVL2emc7lT5ik=
 github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4=
 github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
 github.com/itchyny/gojq v0.12.14 h1:6k8vVtsrhQSYgSGg827AD+PVVaB1NLXEdX+dda2oZCc=
 github.com/itchyny/gojq v0.12.14/go.mod h1:y1G7oO7XkcR1LPZO59KyoCRy08T3j9vDYRV0GgYSS+s=
+github.com/itchyny/gojq v0.12.17 h1:8av8eGduDb5+rvEdaOO+zQUjA04MS0m3Ps8HiD+fceg=
+github.com/itchyny/gojq v0.12.17/go.mod h1:WBrEMkgAfAGO1LUcGOckBl5O726KPp+OlkKug0I/FEY=
 github.com/itchyny/timefmt-go v0.1.5 h1:G0INE2la8S6ru/ZI5JecgyzbbJNs5lG1RcBqa7Jm6GE=
 github.com/itchyny/timefmt-go v0.1.5/go.mod h1:nEP7L+2YmAbT2kZ2HfSs1d8Xtw9LY8D2stDBckWakZ8=
+github.com/itchyny/timefmt-go v0.1.6 h1:ia3s54iciXDdzWzwaVKXZPbiXzxxnv1SPGFfM/myJ5Q=
+github.com/itchyny/timefmt-go v0.1.6/go.mod h1:RRDZYC5s9ErkjQvTvvU7keJjxUYzIISJGxm9/mAERQg=
 github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
 github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
 github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk=
 github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
+github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
 github.com/jackc/pgx/v5 v5.5.1 h1:5I9etrGkLrN+2XPCsi6XLlV5DITbSL/xBZdmAxFcXPI=
 github.com/jackc/pgx/v5 v5.5.1/go.mod h1:Ig06C2Vu0t5qXC60W8sqIthScaEnFvojjj9dSljmHRA=
+github.com/jackc/pgx/v5 v5.7.4 h1:9wKznZrhWa2QiHL+NjTSPP6yjl3451BX3imWDnokYlg=
 github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
 github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
+github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
 github.com/jarcoal/httpmock v1.3.1 h1:iUx3whfZWVf3jT01hQTO/Eo5sAYtB2/rqaUuOtpInww=
 github.com/jarcoal/httpmock v1.3.1/go.mod h1:3yb8rc4BI7TCBhFY8ng0gjuLKJNquuDNiPaZjnENuYg=
 github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4=
 github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak=
 github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4=
 github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc=
+github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
 github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4=
 github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc=
 github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
+github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
+github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
 github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
@@ -154,13 +206,22 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY=
 github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
 github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4=
+github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
+github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
 github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U=
 github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/maxatome/go-testdeep v1.12.0 h1:Ql7Go8Tg0C1D/uMMX59LAoYK7LffeJQ6X2T04nTH68g=
+github.com/maxatome/go-testdeep v1.12.0/go.mod h1:lPZc/HAcJMP92l7yI6TRz1aZN5URwUBUAfUNvrclaNM=
+github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY=
+github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg=
 github.com/microcosm-cc/bluemonday v1.0.26 h1:xbqSvqzQMeEHCqMi64VAs4d8uy6Mequs3rQ0k/Khz58=
 github.com/microcosm-cc/bluemonday v1.0.26/go.mod h1:JyzOCs9gkyQyjs+6h10UEVSe02CGwkhd72Xdqh78TWs=
+github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk=
+github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
@@ -170,6 +231,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
+github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.1.0-rc5 h1:Ygwkfw9bpDvs+c9E34SdgGOj41dX/cbdlwvlWt0pnFI=
@@ -180,11 +243,15 @@ github.com/ory/dockertest/v3 v3.10.0 h1:4K3z2VMe8Woe++invjaTB7VRyQXQy5UY+loujO4a
 github.com/ory/dockertest/v3 v3.10.0/go.mod h1:nr57ZbRWMqfsdGdFNLHz5jjNdDb7VVFnzAeW1n5N1Lg=
 github.com/paulmach/orb v0.10.0 h1:guVYVqzxHE/CQ1KpfGO077TR0ATHSNjp4s6XGLn3W9s=
 github.com/paulmach/orb v0.10.0/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU=
+github.com/paulmach/orb v0.11.1 h1:3koVegMC4X/WeiXYz9iswopaTwMem53NzTJuTF20JzU=
 github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo=
 github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI=
 github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
+github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
+github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
 github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ=
 github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
+github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -192,21 +259,28 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/pressly/goose/v3 v3.17.0 h1:fT4CL3LRm4kfyLuPWzDFAoxjR5ZHjeJ6uQhibQtBaIs=
 github.com/pressly/goose/v3 v3.17.0/go.mod h1:22aw7NpnCPlS86oqkO/+3+o9FuCaJg4ZVWRUO3oGzHQ=
+github.com/pressly/goose/v3 v3.24.3 h1:DSWWNwwggVUsYZ0X2VitiAa9sKuqtBfe+Jr9zFGwWlM=
+github.com/pressly/goose/v3 v3.24.3/go.mod h1:v9zYL4xdViLHCUUJh/mhjnm6JrK7Eul8AS93IxiZM4E=
 github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
 github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
+github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
 github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8=
 github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
+github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys=
 github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=
 github.com/sethvargo/go-retry v0.2.4 h1:T+jHEQy/zKJf5s95UkguisicE0zuF9y7+/vgz08Ocec=
 github.com/sethvargo/go-retry v0.2.4/go.mod h1:1afjQuvh7s4gflMObvjLPaWgluLLyhA1wmVZ6KLpICw=
+github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE=
+github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas=
 github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
 github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
+github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -221,16 +295,24 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o
 github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/taylorskalyo/goreader v0.0.0-20230626212555-e7f5644f8115 h1:OEAIMYp5l9kJ2kT9UPL5QSUriKIIDhnLmpJTy69sltA=
 github.com/taylorskalyo/goreader v0.0.0-20230626212555-e7f5644f8115/go.mod h1:AIVbkIe1G7fpFHiKOdxZnU5p9tFPYNTQyH3H5IrRkGw=
+github.com/taylorskalyo/goreader v1.0.1 h1:eS9SYiHai2aAHhm+YMGRTqrvNt2aoRMTd7p6ftm0crY=
+github.com/taylorskalyo/goreader v1.0.1/go.mod h1:JrUsWCgnk4C3P5Jsr7Pf2mFrMpsR0ls/0bjR5aorYTI=
 github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
 github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
 github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M=
 github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY=
 github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE=
 github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
+github.com/ugorji/go/codec v1.3.0 h1:Qd2W2sQawAfG8XSvzwhBeoGq71zXOC/Q1E9y/wUcsUA=
+github.com/ugorji/go/codec v1.3.0/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4=
 github.com/urfave/cli/v2 v2.27.1 h1:8xSQ6szndafKVRmfyeUMxkNUJQMjL1F2zmsZ+qHpfho=
 github.com/urfave/cli/v2 v2.27.1/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ=
+github.com/urfave/cli/v2 v2.27.7 h1:bH59vdhbjLv3LAvIu6gd0usJHgoTTPhCFib8qqOwXYU=
+github.com/urfave/cli/v2 v2.27.7/go.mod h1:CyNAG/xg+iAOg0N4MPGZqVmv2rCoP267496AOXUZjA4=
 github.com/vertica/vertica-sql-go v1.3.3 h1:fL+FKEAEy5ONmsvya2WH5T8bhkvY27y/Ik3ReR2T+Qw=
 github.com/vertica/vertica-sql-go v1.3.3/go.mod h1:jnn2GFuv+O2Jcjktb7zyc4Utlbu9YVqpHH/lx63+1M4=
 github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo=
@@ -241,32 +323,53 @@ github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17
 github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
 github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e h1:+SOyEddqYF09QP7vr7CgJ1eti3pY9Fn3LHO1M1r/0sI=
 github.com/xrash/smetrics v0.0.0-20231213231151-1d8dd44e695e/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
+github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342 h1:FnBeRrxr7OU4VvAzt5X7s6266i6cSVkkFPS0TuXWbIg=
+github.com/xrash/smetrics v0.0.0-20250705151800-55b8f293f342/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM=
 github.com/ydb-platform/ydb-go-genproto v0.0.0-20231012155159-f85a672542fd h1:dzWP1Lu+A40W883dK/Mr3xyDSM/2MggS8GtHT0qgAnE=
 github.com/ydb-platform/ydb-go-genproto v0.0.0-20231012155159-f85a672542fd/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
+github.com/ydb-platform/ydb-go-genproto v0.0.0-20241112172322-ea1f63298f77 h1:LY6cI8cP4B9rrpTleZk95+08kl2gF4rixG7+V/dwL6Q=
 github.com/ydb-platform/ydb-go-sdk/v3 v3.54.2 h1:E0yUuuX7UmPxXm92+yQCjMveLFO3zfvYFIJVuAqsVRA=
 github.com/ydb-platform/ydb-go-sdk/v3 v3.54.2/go.mod h1:fjBLQ2TdQNl4bMjuWl9adoTGBypwUTPoGC+EqYqiIcU=
+github.com/ydb-platform/ydb-go-sdk/v3 v3.108.1 h1:ixAiqjj2S/dNuJqrz4AxSqgw2P5OBMXp68hB5nNriUk=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 go.opentelemetry.io/otel v1.20.0 h1:vsb/ggIY+hUjD/zCAQHpzTmndPqv/ml2ArbsbfBYTAc=
 go.opentelemetry.io/otel v1.20.0/go.mod h1:oUIGj3D77RwJdM6PPZImDpSZGDvkD9fhesHny69JFrs=
+go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ=
 go.opentelemetry.io/otel/trace v1.20.0 h1:+yxVAPZPbQhbC3OfAkeIVTky6iTFpcr4SiY9om7mXSQ=
 go.opentelemetry.io/otel/trace v1.20.0/go.mod h1:HJSK7F/hA5RlzpZ0zKDCHCDHm556LCDtKaAo6JmBFUU=
+go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs=
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
 go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
 golang.org/x/arch v0.7.0 h1:pskyeJh/3AmoQ8CPE95vxHLqp1G1GfGNXTmcl9NEKTc=
 golang.org/x/arch v0.7.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys=
+golang.org/x/arch v0.20.0 h1:dx1zTU0MAE98U+TQ8BLl7XsJbgze2WnNKF/8tGp/Q6c=
+golang.org/x/arch v0.20.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
 golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
 golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc=
 golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=
+golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
+golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
+golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
+golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4=
+golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc=
 golang.org/x/exp v0.0.0-20240119083558-1b970713d09a h1:Q8/wZp0KX97QFTc2ywcOE0YRjZPVIx+MXInMzdvQqcA=
 golang.org/x/exp v0.0.0-20240119083558-1b970713d09a/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08=
+golang.org/x/exp v0.0.0-20250718183923-645b1fa84792 h1:R9PFI6EUdfVKgwKjZef7QIwGcBKu86OEFpJ9nUEP2l4=
+golang.org/x/exp v0.0.0-20250718183923-645b1fa84792/go.mod h1:A+z0yzpGtvnG90cToK5n2tu8UJVP2XUATh+r+sfOOOc=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
 golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ=
+golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
@@ -275,13 +378,24 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
 golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo=
 golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
+golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
+golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
+golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE=
+golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
 golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
 golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw=
+golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -295,15 +409,26 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
 golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI=
+golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
 golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
+golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
 golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
+golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
+golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
+golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -313,22 +438,35 @@ golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
+golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng=
+golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
 golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc=
 golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps=
+golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
+golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg=
+golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 h1:Jyp0Hsi0bmHXG6k9eATXoYtjd6e2UzZ1SCn/wIupY14=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:oQ5rr10WTTMvP4A36n8JpR1OrO1BEiV4f78CneXZxkA=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20250324211829-b45e905df463 h1:e0AIkUUhxyBKh6ssZNrAMeqhA7RKUj42346d1y02i2g=
 google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk=
 google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98=
+google.golang.org/grpc v1.71.0 h1:kF77BGdPTQ4/JZWMlb9VpJ5pa25aqvVqogsxNHHdeBg=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I=
 google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
+google.golang.org/protobuf v1.36.7 h1:IgrO7UwFQGJdRNXH/sQux4R1Dj1WAKcLElzeeRaXV2A=
+google.golang.org/protobuf v1.36.7/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
@@ -344,28 +482,43 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 howett.net/plist v1.0.0 h1:7CrbWYbPPO/PyNy38b2EB/+gYbjCe2DXBxgtOOZbSQM=
 howett.net/plist v1.0.0/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g=
+howett.net/plist v1.0.1 h1:37GdZ8tP09Q35o9ych3ehygcsL+HqKSwzctveSlarvM=
 lukechampine.com/uint128 v1.3.0 h1:cDdUVfRwDUDovz610ABgFD17nXD4/uDgVHl2sC3+sbo=
 lukechampine.com/uint128 v1.3.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=
 modernc.org/cc/v3 v3.41.0 h1:QoR1Sn3YWlmA1T4vLaKZfawdVtSiGx8H+cEojbC7v1Q=
 modernc.org/cc/v3 v3.41.0/go.mod h1:Ni4zjJYJ04CDOhG7dn640WGfwBzfE0ecX8TyMB0Fv0Y=
 modernc.org/ccgo/v3 v3.16.15 h1:KbDR3ZAVU+wiLyMESPtbtE/Add4elztFyfsWoNTgxS0=
 modernc.org/ccgo/v3 v3.16.15/go.mod h1:yT7B+/E2m43tmMOT51GMoM98/MtHIcQQSleGnddkUNI=
+modernc.org/ccgo/v3 v3.17.0 h1:o3OmOqx4/OFnl4Vm3G8Bgmqxnvxnh0nbxeT5p/dWChA=
+modernc.org/ccgo/v3 v3.17.0/go.mod h1:Sg3fwVpmLvCUTaqEUjiBDAvshIaKDB0RXaf+zgqFu8I=
 modernc.org/ccorpus v1.11.6 h1:J16RXiiqiCgua6+ZvQot4yUuUy8zxgqbqEEUuGPlISk=
 modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ=
 modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM=
 modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM=
 modernc.org/libc v1.40.7 h1:oeLS0G067ZqUu+v143Dqad0btMfKmNS7SuOsnkq0Ysg=
 modernc.org/libc v1.40.7/go.mod h1:YAXkAZ8ktnkCKaN9sw/UDeUVkGYJ/YquGO4FTi5nmHE=
+modernc.org/libc v1.66.6 h1:RyQpwAhM/19nXD8y3iejM/AjmKwY2TjxZTlUWTsWw2U=
+modernc.org/libc v1.66.6/go.mod h1:j8z0EYAuumoMQ3+cWXtmw6m+LYn3qm8dcZDFtFTSq+M=
 modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4=
 modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo=
+modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
+modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
 modernc.org/memory v1.7.2 h1:Klh90S215mmH8c9gO98QxQFsY+W451E8AnzjoE2ee1E=
 modernc.org/memory v1.7.2/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E=
+modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
+modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
 modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4=
 modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
+modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8=
+modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
 modernc.org/sqlite v1.28.0 h1:Zx+LyDDmXczNnEQdvPuEfcFVA2ZPyaD7UCZDjef3BHQ=
 modernc.org/sqlite v1.28.0/go.mod h1:Qxpazz0zH8Z1xCFyi5GSL3FzbtZ3fvbjmywNogldEW0=
+modernc.org/sqlite v1.38.2 h1:Aclu7+tgjgcQVShZqim41Bbw9Cho0y/7WzYptXqkEek=
+modernc.org/sqlite v1.38.2/go.mod h1:cPTJYSlgg3Sfg046yBShXENNtPrWrDX8bsbAQBzgQ5E=
 modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA=
 modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0=
+modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
+modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
 modernc.org/tcl v1.15.2 h1:C4ybAYCGJw968e+Me18oW55kD/FexcHbqH2xak1ROSY=
 modernc.org/tcl v1.15.2/go.mod h1:3+k/ZaEbKrC8ePv8zJWPtBSW0V7Gg9g8rkmhI1Kfs3c=
 modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=

metadata/epub.go

@@ -53,10 +53,12 @@ func countEPUBWords(filepath string) (int64, error) {
 	rf := rc.Rootfiles[0]
 
 	var completeCount int64
-	for _, item := range rf.Spine.Itemrefs {
+	for _, item := range rf.Itemrefs {
 		f, _ := item.Open()
 		doc, _ := goquery.NewDocumentFromReader(f)
-		completeCount = completeCount + int64(len(strings.Fields(doc.Text())))
+		doc.Find("script, style, noscript, iframe").Remove()
+		words := len(strings.Fields(doc.Text()))
+		completeCount = completeCount + int64(words)
 	}
 
 	return completeCount, nil

metadata/metadata.go

@@ -87,7 +87,7 @@ func GetWordCount(filepath string) (*int64, error) {
 		}
 		return &totalWords, nil
 	} else {
-		return nil, fmt.Errorf("Invalid extension: %s", fileExtension)
+		return nil, fmt.Errorf("invalid extension: %s", fileExtension)
 	}
 }
 

metadata/metadata_test.go

@@ -8,7 +8,7 @@ import (
 )
 
 func TestGetWordCount(t *testing.T) {
-	var desiredCount int64 = 30080
+	var desiredCount int64 = 30070
 	actualCount, err := countEPUBWords("../_test_files/alice.epub")
 
 	assert.Nil(t, err, "should have no error")

pkg/formatters/duration.go

@@ -0,0 +1,37 @@
+package formatters
+
+import (
+	"fmt"
+	"strings"
+	"time"
+)
+
+// FormatDuration takes a duration and returns a human-readable duration string.
+// For example: 1928371 seconds -> "22d 7h 39m 31s"
+func FormatDuration(d time.Duration) string {
+	if d == 0 {
+		return "N/A"
+	}
+
+	var parts []string
+
+	days := int(d.Hours()) / 24
+	hours := int(d.Hours()) % 24
+	minutes := int(d.Minutes()) % 60
+	seconds := int(d.Seconds()) % 60
+
+	if days > 0 {
+		parts = append(parts, fmt.Sprintf("%dd", days))
+	}
+	if hours > 0 {
+		parts = append(parts, fmt.Sprintf("%dh", hours))
+	}
+	if minutes > 0 {
+		parts = append(parts, fmt.Sprintf("%dm", minutes))
+	}
+	if seconds > 0 {
+		parts = append(parts, fmt.Sprintf("%ds", seconds))
+	}
+
+	return strings.Join(parts, " ")
+}

pkg/formatters/duration_test.go

@@ -0,0 +1,22 @@
+package formatters
+
+import (
+    "testing"
+    "time"
+)
+
+func TestFormatDuration(t *testing.T) {
+    tests := []struct {
+        dur  time.Duration
+        want string
+    }{
+        {0, "N/A"},
+        {22*24*time.Hour + 7*time.Hour + 39*time.Minute + 31*time.Second, "22d 7h 39m 31s"},
+        {5*time.Minute + 15*time.Second, "5m 15s"},
+    }
+    for _, tc := range tests {
+        if got := FormatDuration(tc.dur); got != tc.want {
+            t.Errorf("FormatDuration(%v) = %s, want %s", tc.dur, got, tc.want)
+        }
+    }
+}

pkg/formatters/numbers.go

@@ -0,0 +1,45 @@
+package formatters
+
+import (
+	"fmt"
+	"math"
+)
+
+// FormatNumber takes an int64 and returns a human-readable string.
+// For example: 19823 -> "19.8k", 1500000 -> "1.5M"
+func FormatNumber(input int64) string {
+	if input == 0 {
+		return "0"
+	}
+
+	// Handle Negative
+	negative := input < 0
+	if negative {
+		input = -input
+	}
+
+	abbreviations := []string{"", "k", "M", "B", "T"}
+	abbrevIndex := int(math.Log10(float64(input)) / 3)
+
+	// Bounds Check
+	if abbrevIndex >= len(abbreviations) {
+		abbrevIndex = len(abbreviations) - 1
+	}
+
+	scaledNumber := float64(input) / math.Pow(10, float64(abbrevIndex*3))
+
+	var result string
+	if scaledNumber >= 100 {
+		result = fmt.Sprintf("%.0f%s", scaledNumber, abbreviations[abbrevIndex])
+	} else if scaledNumber >= 10 {
+		result = fmt.Sprintf("%.1f%s", scaledNumber, abbreviations[abbrevIndex])
+	} else {
+		result = fmt.Sprintf("%.2f%s", scaledNumber, abbreviations[abbrevIndex])
+	}
+
+	if negative {
+		result = "-" + result
+	}
+
+	return result
+}

pkg/formatters/numbers_test.go

@@ -0,0 +1,22 @@
+package formatters
+
+import (
+    "testing"
+)
+
+func TestFormatNumber(t *testing.T) {
+    tests := []struct {
+        input int64
+        want  string
+    }{
+        {0, "0"},
+        {19823, "19.8k"},
+        {1500000, "1.50M"},
+        {-12345, "-12.3k"},
+    }
+    for _, tc := range tests {
+        if got := FormatNumber(tc.input); got != tc.want {
+            t.Errorf("FormatNumber(%d) = %s, want %s", tc.input, got, tc.want)
+        }
+    }
+}

pkg/ptr/ptr.go

@@ -0,0 +1,13 @@
+package ptr
+
+func Of[T any](v T) *T {
+	return &v
+}
+
+func Deref[T any](v *T) T {
+	var zeroT T
+	if v == nil {
+		return zeroT
+	}
+	return *v
+}

pkg/ptr/ptr_test.go

@@ -0,0 +1,73 @@
+package ptr
+
+import (
+	"testing"
+)
+
+func TestOf(t *testing.T) {
+	// Test with different types
+	intVal := 42
+	intPtr := Of(intVal)
+	if *intPtr != intVal {
+		t.Errorf("Expected %d, got %d", intVal, *intPtr)
+	}
+
+	stringVal := "hello"
+	stringPtr := Of(stringVal)
+	if *stringPtr != stringVal {
+		t.Errorf("Expected %s, got %s", stringVal, *stringPtr)
+	}
+
+	floatVal := 3.14
+	floatPtr := Of(floatVal)
+	if *floatPtr != floatVal {
+		t.Errorf("Expected %f, got %f", floatVal, *floatPtr)
+	}
+}
+
+func TestDeref(t *testing.T) {
+	// Test with non-nil pointer
+	intVal := 42
+	intPtr := Of(intVal)
+	result := Deref(intPtr)
+	if result != intVal {
+		t.Errorf("Expected %d, got %d", intVal, result)
+	}
+
+	// Test with nil pointer
+	var nilPtr *int
+	result = Deref(nilPtr)
+	if result != 0 {
+		t.Errorf("Expected 0, got %d", result)
+	}
+
+	// Test with string
+	stringVal := "hello"
+	stringPtr := Of(stringVal)
+	resultStr := Deref(stringPtr)
+	if resultStr != stringVal {
+		t.Errorf("Expected %s, got %s", stringVal, resultStr)
+	}
+
+	// Test with nil string pointer
+	var nilStrPtr *string
+	resultStr = Deref(nilStrPtr)
+	if resultStr != "" {
+		t.Errorf("Expected empty string, got %s", resultStr)
+	}
+}
+
+func TestDerefZeroValue(t *testing.T) {
+	// Test that Deref returns zero value for nil pointers
+	var nilInt *int
+	result := Deref(nilInt)
+	if result != 0 {
+		t.Errorf("Expected zero int, got %d", result)
+	}
+
+	var nilString *string
+	resultStr := Deref(nilString)
+	if resultStr != "" {
+		t.Errorf("Expected zero string, got %s", resultStr)
+	}
+}

pkg/sliceutils/sliceutils.go

@@ -0,0 +1,17 @@
+package sliceutils
+
+func First[T any](s []T) (T, bool) {
+	if len(s) == 0 {
+		var zeroT T
+		return zeroT, false
+	}
+	return s[0], true
+}
+
+func Map[R, I any](s []I, f func(I) R) []R {
+	r := make([]R, 0, len(s))
+	for _, v := range s {
+		r = append(r, f(v))
+	}
+	return r
+}

pkg/sliceutils/sliceutils_test.go

@@ -0,0 +1,50 @@
+package sliceutils
+
+import (
+	"testing"
+)
+
+func TestFirst(t *testing.T) {
+	// Test with empty slice
+	var empty []int
+	result, ok := First(empty)
+	if ok != false {
+		t.Errorf("Expected ok=false for empty slice, got %v", ok)
+	}
+	if result != 0 {
+		t.Errorf("Expected zero value for empty slice, got %v", result)
+	}
+
+	// Test with non-empty slice
+	testSlice := []int{1, 2, 3}
+	result, ok = First(testSlice)
+	if ok != true {
+		t.Errorf("Expected ok=true for non-empty slice, got %v", ok)
+	}
+	if result != 1 {
+		t.Errorf("Expected first element, got %v", result)
+	}
+}
+
+func TestMap(t *testing.T) {
+	// Test with empty slice
+	var empty []int
+	result := Map(empty, func(x int) int { return x * 2 })
+	if len(result) != 0 {
+		t.Errorf("Expected empty result for empty input, got %v", result)
+	}
+
+	// Test with non-empty slice
+	testSlice := []int{1, 2, 3}
+	result = Map(testSlice, func(x int) int { return x * 2 })
+	expected := []int{2, 4, 6}
+	if len(result) != len(expected) {
+		t.Errorf("Expected length %d, got %d", len(expected), len(result))
+	}
+	for i, v := range result {
+		if v != expected[i] {
+			t.Errorf("Expected %d at index %d, got %d", expected[i], i, v)
+		}
+	}
+}
+

pkg/utils/utils.go

@@ -0,0 +1,18 @@
+package utils
+
+func Ternary[T any](cond bool, tVal, fVal T) T {
+	if cond {
+		return tVal
+	}
+	return fVal
+}
+
+func FirstNonZero[T comparable](v ...T) T {
+	var zero T
+	for _, val := range v {
+		if val != zero {
+			return val
+		}
+	}
+	return zero
+}

pkg/utils/utils_test.go

@@ -0,0 +1,45 @@
+package utils
+
+import (
+	"testing"
+)
+
+func TestTernary(t *testing.T) {
+	// Test true condition
+	result := Ternary(true, 42, 13)
+	if result != 42 {
+		t.Errorf("Expected 42, got %d", result)
+	}
+
+	// Test false condition
+	result = Ternary(false, 42, 13)
+	if result != 13 {
+		t.Errorf("Expected 13, got %d", result)
+	}
+}
+
+func TestFirstNonZero(t *testing.T) {
+	// Test with int values
+	result := FirstNonZero(0, 0, 42, 13)
+	if result != 42 {
+		t.Errorf("Expected 42, got %d", result)
+	}
+
+	// Test with string values
+	resultStr := FirstNonZero("", "", "hello")
+	if resultStr != "hello" {
+		t.Errorf("Expected hello, got %s", resultStr)
+	}
+
+	// Test all zero values (strings)
+	zeroResultStr := FirstNonZero("")
+	if zeroResultStr != "" {
+		t.Errorf("Expected empty string, got %s", zeroResultStr)
+	}
+
+	// Test with float values
+	floatResult := FirstNonZero(0.0, 0.0, 3.14)
+	if floatResult != 3.14 {
+		t.Errorf("Expected 3.14, got %f", floatResult)
+	}
+}

search/anna.go

@@ -3,58 +3,20 @@ package search
 import (
 	"fmt"
 	"io"
-	"regexp"
+	"net/url"
 	"strings"
 
 	"github.com/PuerkitoBio/goquery"
 )
 
-var commentRE = regexp.MustCompile(`(?s)<!--(.*?)-->`)
-
-func parseAnnasArchiveDownloadURL(body io.ReadCloser) (string, error) {
-	// Parse
-	defer body.Close()
-	doc, _ := goquery.NewDocumentFromReader(body)
-
-	// Return Download URL
-	downloadPath, exists := doc.Find("body > table > tbody > tr > td > a").Attr("href")
-	if !exists {
-		return "", fmt.Errorf("Download URL not found")
-	}
-
-	// Possible Funky URL
-	downloadPath = strings.ReplaceAll(downloadPath, "\\", "/")
-
-	return fmt.Sprintf("http://libgen.li/%s", downloadPath), nil
-}
-
-// getAnnasArchiveBookSelection parses potentially commented out HTML. For some reason
-// Annas Archive comments out blocks "below the fold". They aren't rendered until you
-// scroll. This attempts to parse the commented out HTML.
-func getAnnasArchiveBookSelection(rawBook *goquery.Selection) *goquery.Selection {
-	rawHTML, err := rawBook.Html()
+func searchAnnasArchive(query string) ([]SearchItem, error) {
+	searchURL := "https://annas-archive.org/search?index=&q=%s&ext=epub&sort=&lang=en"
+	url := fmt.Sprintf(searchURL, url.QueryEscape(query))
+	body, err := getPage(url)
 	if err != nil {
-		return rawBook
+		return nil, err
 	}
-
-	strippedHTML := strings.TrimSpace(rawHTML)
-	if !strings.HasPrefix(strippedHTML, "<!--") || !strings.HasSuffix(strippedHTML, "-->") {
-		return rawBook
-	}
-
-	allMatches := commentRE.FindAllStringSubmatch(strippedHTML, -1)
-	if len(allMatches) != 1 || len(allMatches[0]) != 2 {
-		return rawBook
-	}
-
-	captureGroup := allMatches[0][1]
-	docReader := strings.NewReader(captureGroup)
-	doc, err := goquery.NewDocumentFromReader(docReader)
-	if err != nil {
-		return rawBook
-	}
-
-	return doc.Selection
+	return parseAnnasArchive(body)
 }
 
 func parseAnnasArchive(body io.ReadCloser) ([]SearchItem, error) {
@@ -67,41 +29,32 @@ func parseAnnasArchive(body io.ReadCloser) ([]SearchItem, error) {
 
 	// Normalize Results
 	var allEntries []SearchItem
-	doc.Find("form > div.w-full > div.w-full > div > div.justify-center").Each(func(ix int, rawBook *goquery.Selection) {
-		rawBook = getAnnasArchiveBookSelection(rawBook)
+	doc.Find(".js-aarecord-list-outer > div > div").Each(func(ix int, rawBook *goquery.Selection) {
 
 		// Parse Details
-		details := rawBook.Find("div:nth-child(2) > div:nth-child(1)").Text()
-		detailsSplit := strings.Split(details, ", ")
+		details := rawBook.Find("div:nth-child(3)").Text()
+		detailsSplit := strings.Split(details, " · ")
 
 		// Invalid Details
-		if len(detailsSplit) < 4 {
+		if len(detailsSplit) < 3 {
 			return
 		}
 
-		language := detailsSplit[0]
-		fileType := detailsSplit[1]
-		fileSize := detailsSplit[3]
-
-		// Get Title & Author
-		title := rawBook.Find("h3").Text()
-		author := rawBook.Find("div:nth-child(2) > div:nth-child(4)").Text()
-
 		// Parse MD5
-		itemHref, _ := rawBook.Find("a").Attr("href")
+		titleAuthorDetails := rawBook.Find("div a")
+		titleEl := titleAuthorDetails.Eq(0)
+		itemHref, _ := titleEl.Attr("href")
 		hrefArray := strings.Split(itemHref, "/")
 		id := hrefArray[len(hrefArray)-1]
 
-		item := SearchItem{
+		allEntries = append(allEntries, SearchItem{
 			ID:       id,
-			Title:    title,
-			Author:   author,
-			Language: language,
-			FileType: fileType,
-			FileSize: fileSize,
-		}
-
-		allEntries = append(allEntries, item)
+			Title:    titleEl.Text(),
+			Author:   titleAuthorDetails.Eq(1).Text(),
+			Language: detailsSplit[0],
+			FileType: detailsSplit[1],
+			FileSize: detailsSplit[2],
+		})
 	})
 
 	// Return Results

search/downloaders.go

@@ -0,0 +1,69 @@
+package search
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+func getLibGenDownloadURL(md5 string, _ Source) ([]string, error) {
+	// Get Page
+	body, err := getPage("http://libgen.li/ads.php?md5=" + md5)
+	if err != nil {
+		return nil, err
+	}
+	defer body.Close()
+
+	// Parse
+	doc, err := goquery.NewDocumentFromReader(body)
+	if err != nil {
+		return nil, err
+	}
+
+	// Return Download URL
+	downloadPath, exists := doc.Find("body > table > tbody > tr > td > a").Attr("href")
+	if !exists {
+		return nil, fmt.Errorf("download URL not found")
+	}
+
+	// Possible Funky URL
+	downloadPath = strings.ReplaceAll(downloadPath, "\\", "/")
+	return []string{fmt.Sprintf("http://libgen.li/%s", downloadPath)}, nil
+}
+
+func getLibraryDownloadURL(md5 string, source Source) ([]string, error) {
+	// Derive Info URL
+	var infoURL string
+	switch source {
+	case SOURCE_LIBGEN, SOURCE_ANNAS_ARCHIVE:
+		infoURL = "http://library.lol/fiction/" + md5
+	// case SOURCE_LIBGEN_NON_FICTION:
+	// 	infoURL = "http://library.lol/main/" + md5
+	default:
+		return nil, errors.New("invalid source")
+	}
+
+	// Get Page
+	body, err := getPage(infoURL)
+	if err != nil {
+		return nil, err
+	}
+	defer body.Close()
+
+	// Parse
+	doc, err := goquery.NewDocumentFromReader(body)
+	if err != nil {
+		return nil, err
+	}
+
+	// Return Download URL
+	// downloadURL, _ := doc.Find("#download [href*=cloudflare]").Attr("href")
+	downloadURL, exists := doc.Find("#download h2 a").Attr("href")
+	if !exists {
+		return nil, errors.New("download URL not found")
+	}
+
+	return []string{downloadURL}, nil
+}

search/libgen.go

@@ -1,68 +1,44 @@
 package search
 
 import (
+	"errors"
 	"fmt"
 	"io"
+	"net/url"
 	"strings"
-	"time"
 
 	"github.com/PuerkitoBio/goquery"
 )
 
-func parseLibGenFiction(body io.ReadCloser) ([]SearchItem, error) {
-	// Parse
-	defer body.Close()
-	doc, err := goquery.NewDocumentFromReader(body)
-	if err != nil {
-		return nil, err
-	}
+const LIBGEN_SEARCH_URL = "https://%s/index.php?req=ext:epub+%s&gmode=on"
 
-	// Normalize Results
-	var allEntries []SearchItem
-	doc.Find("table.catalog tbody > tr").Each(func(ix int, rawBook *goquery.Selection) {
-
-		// Parse File Details
-		fileItem := rawBook.Find("td:nth-child(5)")
-		fileDesc := fileItem.Text()
-		fileDescSplit := strings.Split(fileDesc, "/")
-		fileType := strings.ToLower(strings.TrimSpace(fileDescSplit[0]))
-		fileSize := strings.TrimSpace(fileDescSplit[1])
-
-		// Parse Upload Date
-		uploadedRaw, _ := fileItem.Attr("title")
-		uploadedDateRaw := strings.Split(uploadedRaw, "Uploaded at ")[1]
-		uploadDate, _ := time.Parse("2006-01-02 15:04:05", uploadedDateRaw)
-
-		// Parse MD5
-		editHref, _ := rawBook.Find("td:nth-child(7) a").Attr("href")
-		hrefArray := strings.Split(editHref, "/")
-		id := hrefArray[len(hrefArray)-1]
-
-		// Parse Other Details
-		title := rawBook.Find("td:nth-child(3) p a").Text()
-		author := rawBook.Find(".catalog_authors li a").Text()
-		language := rawBook.Find("td:nth-child(4)").Text()
-		series := rawBook.Find("td:nth-child(2)").Text()
-
-		item := SearchItem{
-			ID:         id,
-			Title:      title,
-			Author:     author,
-			Series:     series,
-			Language:   language,
-			FileType:   fileType,
-			FileSize:   fileSize,
-			UploadDate: uploadDate.Format(time.RFC3339),
-		}
-
-		allEntries = append(allEntries, item)
-	})
-
-	// Return Results
-	return allEntries, nil
+var libgenDomains []string = []string{
+	"libgen.vg",
+	"libgen.is",
 }
 
-func parseLibGenNonFiction(body io.ReadCloser) ([]SearchItem, error) {
+func searchLibGen(query string) ([]SearchItem, error) {
+	var allErrors []error
+
+	for _, domain := range libgenDomains {
+		url := fmt.Sprintf(LIBGEN_SEARCH_URL, domain, url.QueryEscape(query))
+		body, err := getPage(url)
+		if err != nil {
+			allErrors = append(allErrors, err)
+			continue
+		}
+		results, err := parseLibGen(body)
+		if err != nil {
+			allErrors = append(allErrors, err)
+			continue
+		}
+		return results, nil
+	}
+
+	return nil, fmt.Errorf("could not query libgen: %w", errors.Join(allErrors...))
+}
+
+func parseLibGen(body io.ReadCloser) ([]SearchItem, error) {
 	// Parse
 	defer body.Close()
 	doc, err := goquery.NewDocumentFromReader(body)
@@ -72,52 +48,27 @@ func parseLibGenNonFiction(body io.ReadCloser) ([]SearchItem, error) {
 
 	// Normalize Results
 	var allEntries []SearchItem
-	doc.Find("table.c tbody > tr:nth-child(n + 2)").Each(func(ix int, rawBook *goquery.Selection) {
-
-		// Parse Type & Size
-		fileSize := strings.ToLower(strings.TrimSpace(rawBook.Find("td:nth-child(8)").Text()))
-		fileType := strings.ToLower(strings.TrimSpace(rawBook.Find("td:nth-child(9)").Text()))
-
+	doc.Find("#tablelibgen tbody > tr").Each(func(ix int, rawBook *goquery.Selection) {
 		// Parse MD5
-		titleRaw := rawBook.Find("td:nth-child(3) [id]")
-		editHref, _ := titleRaw.Attr("href")
-		hrefArray := strings.Split(editHref, "?md5=")
+		linksRaw := rawBook.Find("td:nth-child(9) a")
+		linksHref, _ := linksRaw.Attr("href")
+		hrefArray := strings.Split(linksHref, "?md5=")
+		if len(hrefArray) == 0 {
+			return
+		}
 		id := hrefArray[1]
 
-		// Parse Other Details
-		title := titleRaw.Text()
-		author := rawBook.Find("td:nth-child(2)").Text()
-		language := rawBook.Find("td:nth-child(7)").Text()
-		series := rawBook.Find("td:nth-child(3) [href*='column=series']").Text()
-
-		item := SearchItem{
+		allEntries = append(allEntries, SearchItem{
 			ID:       id,
-			Title:    title,
-			Author:   author,
-			Series:   series,
-			Language: language,
-			FileType: fileType,
-			FileSize: fileSize,
-		}
-
-		allEntries = append(allEntries, item)
+			Title:    rawBook.Find("td:nth-child(1) > a").First().Text(),
+			Author:   rawBook.Find("td:nth-child(2)").Text(),
+			Series:   rawBook.Find("td:nth-child(1) > b").Text(),
+			Language: rawBook.Find("td:nth-child(5)").Text(),
+			FileType: strings.ToLower(strings.TrimSpace(rawBook.Find("td:nth-child(8)").Text())),
+			FileSize: strings.ToLower(strings.TrimSpace(rawBook.Find("td:nth-child(7)").Text())),
+		})
 	})
 
 	// Return Results
 	return allEntries, nil
 }
-
-func parseLibGenDownloadURL(body io.ReadCloser) (string, error) {
-	// Parse
-	defer body.Close()
-	doc, _ := goquery.NewDocumentFromReader(body)
-
-	// Return Download URL
-	// downloadURL, _ := doc.Find("#download [href*=cloudflare]").Attr("href")
-	downloadURL, exists := doc.Find("#download h2 a").Attr("href")
-	if !exists {
-		return "", fmt.Errorf("Download URL not found")
-	}
-
-	return downloadURL, nil
-}

search/progress.go

@@ -0,0 +1,22 @@
+package search
+
+type writeCounter struct {
+	Total            int64
+	Current          int64
+	ProgressFunction func(float32)
+}
+
+func (wc *writeCounter) Write(p []byte) (int, error) {
+	n := len(p)
+	wc.Current += int64(n)
+	wc.flushProgress()
+	return n, nil
+}
+
+func (wc *writeCounter) flushProgress() {
+	if wc.ProgressFunction == nil || wc.Total < 100000 {
+		return
+	}
+	percentage := float32(wc.Current) * 100 / float32(wc.Total)
+	wc.ProgressFunction(percentage)
+}

search/search.go

@@ -2,17 +2,18 @@ package search
 
 import (
 	"crypto/tls"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
-	"net/url"
 	"os"
 	"time"
 
 	log "github.com/sirupsen/logrus"
+	"reichard.io/antholume/metadata"
 )
 
-const userAgent string = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:127.0) Gecko/20100101 Firefox/127.0"
+const userAgent string = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
 
 type Cadence string
 
@@ -21,19 +22,11 @@ const (
 	CADENCE_TOP_MONTH Cadence = "m"
 )
 
-type BookType int
-
-const (
-	BOOK_FICTION BookType = iota
-	BOOK_NON_FICTION
-)
-
 type Source string
 
 const (
-	SOURCE_ANNAS_ARCHIVE      Source = "Annas Archive"
-	SOURCE_LIBGEN_FICTION     Source = "LibGen Fiction"
-	SOURCE_LIBGEN_NON_FICTION Source = "LibGen Non-fiction"
+	SOURCE_ANNAS_ARCHIVE Source = "Annas Archive"
+	SOURCE_LIBGEN        Source = "LibGen"
 )
 
 type SearchItem struct {
@@ -47,108 +40,57 @@ type SearchItem struct {
 	UploadDate string
 }
 
-type sourceDef struct {
-	searchURL         string
-	downloadURL       string
-	parseSearchFunc   func(io.ReadCloser) ([]SearchItem, error)
-	parseDownloadFunc func(io.ReadCloser) (string, error)
+type searchFunc func(query string) (searchResults []SearchItem, err error)
+type downloadFunc func(md5 string, source Source) (downloadURL []string, err error)
+
+var searchDefs = map[Source]searchFunc{
+	SOURCE_ANNAS_ARCHIVE: searchAnnasArchive,
+	SOURCE_LIBGEN:        searchLibGen,
 }
 
-var sourceDefs = map[Source]sourceDef{
-	SOURCE_ANNAS_ARCHIVE: {
-		searchURL:         "https://annas-archive.org/search?index=&q=%s&ext=epub&sort=&lang=en",
-		downloadURL:       "http://libgen.li/ads.php?md5=%s",
-		parseSearchFunc:   parseAnnasArchive,
-		parseDownloadFunc: parseAnnasArchiveDownloadURL,
-	},
-	SOURCE_LIBGEN_FICTION: {
-		searchURL:         "https://libgen.is/fiction/?q=%s&language=English&format=epub",
-		downloadURL:       "http://libgen.li/ads.php?md5=%s",
-		parseSearchFunc:   parseLibGenFiction,
-		parseDownloadFunc: parseAnnasArchiveDownloadURL,
-	},
-	SOURCE_LIBGEN_NON_FICTION: {
-		searchURL:         "https://libgen.is/search.php?req=%s",
-		downloadURL:       "http://libgen.li/ads.php?md5=%s",
-		parseSearchFunc:   parseLibGenNonFiction,
-		parseDownloadFunc: parseAnnasArchiveDownloadURL,
-	},
+var downloadFuncs = []downloadFunc{
+	getLibGenDownloadURL,
+	getLibraryDownloadURL,
 }
 
 func SearchBook(query string, source Source) ([]SearchItem, error) {
-	def := sourceDefs[source]
-	log.Debug("Source: ", def)
-	url := fmt.Sprintf(def.searchURL, url.QueryEscape(query))
-	body, err := getPage(url)
-	if err != nil {
-		return nil, err
+	searchFunc, found := searchDefs[source]
+	if !found {
+		return nil, fmt.Errorf("invalid source: %s", source)
 	}
-	return def.parseSearchFunc(body)
+	log.Debug("Source: ", source)
+	return searchFunc(query)
 }
 
-func SaveBook(id string, source Source) (string, error) {
-	def := sourceDefs[source]
-	log.Debug("Source: ", def)
-	url := fmt.Sprintf(def.downloadURL, id)
+func SaveBook(md5 string, source Source, progressFunc func(float32)) (string, *metadata.MetadataInfo, error) {
+	for _, f := range downloadFuncs {
+		downloadURLs, err := f(md5, source)
+		if err != nil {
+			log.Error("failed to acquire download urls")
+			continue
+		}
 
-	body, err := getPage(url)
-	if err != nil {
-		return "", err
+		for _, bookURL := range downloadURLs {
+			// Download File
+			log.Info("Downloading Book: ", bookURL)
+			fileName, err := downloadBook(bookURL, progressFunc)
+			if err != nil {
+				log.Error("Book URL API Failure: ", err)
+				continue
+			}
+
+			// Get Metadata
+			metadata, err := metadata.GetMetadata(fileName)
+			if err != nil {
+				log.Error("Book Metadata Failure: ", err)
+				continue
+			}
+
+			return fileName, metadata, nil
+		}
 	}
 
-	bookURL, err := def.parseDownloadFunc(body)
-	if err != nil {
-		log.Error("Parse Download URL Error: ", err)
-		return "", fmt.Errorf("Download Failure")
-	}
-
-	// Create File
-	tempFile, err := os.CreateTemp("", "book")
-	if err != nil {
-		log.Error("File Create Error: ", err)
-		return "", fmt.Errorf("File Failure")
-	}
-	defer tempFile.Close()
-
-	// Download File
-	log.Info("Downloading Book: ", bookURL)
-	resp, err := downloadBook(bookURL)
-	if err != nil {
-		os.Remove(tempFile.Name())
-		log.Error("Book URL API Failure: ", err)
-		return "", fmt.Errorf("API Failure")
-	}
-	defer resp.Body.Close()
-
-	// Copy File to Disk
-	log.Info("Saving Book")
-	_, err = io.Copy(tempFile, resp.Body)
-	if err != nil {
-		os.Remove(tempFile.Name())
-		log.Error("File Copy Error: ", err)
-		return "", fmt.Errorf("File Failure")
-	}
-
-	return tempFile.Name(), nil
-}
-
-func GetBookURL(id string, bookType BookType) (string, error) {
-	// Derive Info URL
-	var infoURL string
-	if bookType == BOOK_FICTION {
-		infoURL = "http://library.lol/fiction/" + id
-	} else if bookType == BOOK_NON_FICTION {
-		infoURL = "http://library.lol/main/" + id
-	}
-
-	// Parse & Derive Download URL
-	body, err := getPage(infoURL)
-	if err != nil {
-		return "", err
-	}
-
-	// downloadURL := parseLibGenDownloadURL(body)
-	return parseLibGenDownloadURL(body)
+	return "", nil, errors.New("failed to download book")
 }
 
 func getPage(page string) (io.ReadCloser, error) {
@@ -162,8 +104,6 @@ func getPage(page string) (io.ReadCloser, error) {
 	if err != nil {
 		return nil, err
 	}
-
-	// Set User-Agent
 	req.Header.Set("User-Agent", userAgent)
 
 	// Do Request
@@ -176,7 +116,7 @@ func getPage(page string) (io.ReadCloser, error) {
 	return resp.Body, err
 }
 
-func downloadBook(bookURL string) (*http.Response, error) {
+func downloadBook(bookURL string, progressFunc func(float32)) (string, error) {
 	log.Debug("URL: ", bookURL)
 
 	// Allow Insecure
@@ -189,11 +129,33 @@ func downloadBook(bookURL string) (*http.Response, error) {
 	// Start Request
 	req, err := http.NewRequest("GET", bookURL, nil)
 	if err != nil {
-		return nil, err
+		return "", err
 	}
-
-	// Set User-Agent
 	req.Header.Set("User-Agent", userAgent)
 
-	return client.Do(req)
+	// Perform API Request
+	resp, err := client.Do(req)
+	if err != nil {
+		return "", err
+	}
+
+	// Create File
+	tempFile, err := os.CreateTemp("", "book")
+	if err != nil {
+		log.Error("File Create Error: ", err)
+		return "", fmt.Errorf("failed to create temp file: %w", err)
+	}
+	defer tempFile.Close()
+
+	// Copy File to Disk
+	log.Info("Saving Book")
+	counter := &writeCounter{Total: resp.ContentLength, ProgressFunction: progressFunc}
+	_, err = io.Copy(tempFile, io.TeeReader(resp.Body, counter))
+	if err != nil {
+		os.Remove(tempFile.Name())
+		log.Error("File Copy Error: ", err)
+		return "", fmt.Errorf("failed to copy response to temp file: %w", err)
+	}
+
+	return tempFile.Name(), nil
 }

server/server.go

@@ -1,6 +1,7 @@
 package server
 
 import (
+	"context"
 	"io/fs"
 	"net/http"
 	"sync"
@@ -52,12 +53,14 @@ func (s *server) Start() {
 		ticker := time.NewTicker(15 * time.Minute)
 		defer ticker.Stop()
 
+		ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(5*time.Minute))
 		for {
 			select {
 			case <-ticker.C:
-				s.runScheduledTasks()
+				s.runScheduledTasks(ctx)
 			case <-s.done:
 				log.Info("Stopping task runner...")
+				cancel()
 				return
 			}
 		}
@@ -81,9 +84,9 @@ func (s *server) Stop() {
 }
 
 // Run normal scheduled tasks
-func (s *server) runScheduledTasks() {
+func (s *server) runScheduledTasks(ctx context.Context) {
 	start := time.Now()
-	if err := s.db.CacheTempTables(); err != nil {
+	if err := s.db.CacheTempTables(ctx); err != nil {
 		log.Warn("Refreshing temp table cache failed: ", err)
 	}
 	log.Debug("Completed in: ", time.Since(start))

shell.nix

@@ -1,13 +0,0 @@
-{ pkgs ? import <nixpkgs> { } }:
-
-pkgs.mkShell {
-  packages = with pkgs; [
-    go
-    nodejs
-    nodePackages.tailwindcss
-    python311Packages.grip
-  ];
-  shellHook = ''
-    export PATH=$PATH:~/go/bin
-  '';
-}

templates/pages/search.tmpl

@@ -35,9 +35,8 @@
               id="source"
               name="source"
             >
+              <option value="LibGen">Library Genesis</option>
               <option value="Annas Archive">Annas Archive</option>
-              <option value="LibGen Fiction">LibGen Fiction</option>
-              <option value="LibGen Non-fiction">LibGen Non-fiction</option>
             </select>
           </div>
           <div class="lg:w-60">
@@ -133,7 +132,8 @@
                   </form>
                 </td>
                 <td class="p-3 border-b border-gray-200">
-                  {{ $item.Author }} - {{ $item.Title }}
+                  {{ $item.Author }} -
+                  {{ $item.Title }}
                 </td>
                 <td class="p-3 border-b border-gray-200">
                   <p>{{ or $item.Series "N/A" }}</p>