evan/AnthoLume
evan/AnthoLume
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: evan:425f469097a1891e5e6ba559703b6fed410190ad
Branches Tags
evan:master
evan:templ
evan:0.0.1
..
compare: evan:67dedaa886e2ac07286530db49875ea61c0c47a3
Branches Tags
evan:templ
evan:master
evan:0.0.1

No commits in common. "425f469097a1891e5e6ba559703b6fed410190ad" and "67dedaa886e2ac07286530db49875ea61c0c47a3" have entirely different histories.
425f469097 ... 67dedaa886

16 changed files with 691 additions and 335 deletions
Show Stats Expand all files Collapse all files

26
api/app-routes.go
View File

@ -146,7 +146,7 @@ func (api *API) createAppResourcesRoute(routeName string, args ...map[string]any
} }
templateVars["Data"] = document templateVars["Data"] = document
templateVars["TotalTimeLeftSeconds"] = int64((100.0 - document.Percentage) * float64(document.SecondsPerPercent)) templateVars["TotalTimeLeftSeconds"] = (document.Pages - document.Page) * document.SecondsPerPage
} else if routeName == "activity" { } else if routeName == "activity" {
activityFilter := database.GetActivityParams{ activityFilter := database.GetActivityParams{
UserID: userID, UserID: userID,
@ -177,13 +177,13 @@ func (api *API) createAppResourcesRoute(routeName string, args ...map[string]any
log.Info("GetDatabaseInfo Performance: ", time.Since(start)) log.Info("GetDatabaseInfo Performance: ", time.Since(start))
streaks, _ := api.DB.Queries.GetUserStreaks(api.DB.Ctx, userID) streaks, _ := api.DB.Queries.GetUserStreaks(api.DB.Ctx, userID)
wpm_leaderboard, _ := api.DB.Queries.GetWPMLeaderboard(api.DB.Ctx) wpn_leaderboard, _ := api.DB.Queries.GetWPMLeaderboard(api.DB.Ctx)
templateVars["Data"] = gin.H{ templateVars["Data"] = gin.H{
"Streaks": streaks, "Streaks": streaks,
"GraphData": read_graph_data, "GraphData": read_graph_data,
"DatabaseInfo": database_info, "DatabaseInfo": database_info,
"WPMLeaderboard": wpm_leaderboard, "WPMLeaderboard": wpn_leaderboard,
} }
} else if routeName == "settings" { } else if routeName == "settings" {
user, err := api.DB.Queries.GetUser(api.DB.Ctx, userID) user, err := api.DB.Queries.GetUser(api.DB.Ctx, userID)
@ -456,14 +456,6 @@ func (api *API) uploadNewDocument(c *gin.Context) {
return return
} }
// Get Word Count
wordCount, err := metadata.GetWordCount(tempFile.Name())
if err != nil {
log.Error("[uploadNewDocument] Word Count Failure:", err)
errorPage(c, http.StatusInternalServerError, "Unable to calculate word count.")
return
}
// Derive Filename // Derive Filename
var fileName string var fileName string
if *metadataInfo.Author != "" { if *metadataInfo.Author != "" {
@ -507,7 +499,6 @@ func (api *API) uploadNewDocument(c *gin.Context) {
Title: metadataInfo.Title, Title: metadataInfo.Title,
Author: metadataInfo.Author, Author: metadataInfo.Author,
Description: metadataInfo.Description, Description: metadataInfo.Description,
Words: &wordCount,
Md5: fileHash, Md5: fileHash,
Filepath: &fileName, Filepath: &fileName,
}); err != nil { }); err != nil {
@ -720,7 +711,7 @@ func (api *API) identifyDocument(c *gin.Context) {
} }
templateVars["Data"] = document templateVars["Data"] = document
templateVars["TotalTimeLeftSeconds"] = int64((100.0 - document.Percentage) * float64(document.SecondsPerPercent)) templateVars["TotalTimeLeftSeconds"] = (document.Pages - document.Page) * document.SecondsPerPage
c.HTML(http.StatusOK, "document", templateVars) c.HTML(http.StatusOK, "document", templateVars)
} }
@ -823,14 +814,6 @@ func (api *API) saveNewDocument(c *gin.Context) {
return return
} }
// Get Word Count
wordCount, err := metadata.GetWordCount(safePath)
if err != nil {
log.Error("[saveNewDocument] Word Count Failure:", err)
errorPage(c, http.StatusInternalServerError, "Unable to calculate word count.")
return
}
// Upsert Document // Upsert Document
if _, err = api.DB.Queries.UpsertDocument(api.DB.Ctx, database.UpsertDocumentParams{ if _, err = api.DB.Queries.UpsertDocument(api.DB.Ctx, database.UpsertDocumentParams{
ID: partialMD5, ID: partialMD5,
@ -838,7 +821,6 @@ func (api *API) saveNewDocument(c *gin.Context) {
Author: rDocAdd.Author, Author: rDocAdd.Author,
Md5: fileHash, Md5: fileHash,
Filepath: &fileName, Filepath: &fileName,
Words: &wordCount,
}); err != nil { }); err != nil {
log.Error("[saveNewDocument] UpsertDocument DB Error:", err) log.Error("[saveNewDocument] UpsertDocument DB Error:", err)
errorPage(c, http.StatusInternalServerError, fmt.Sprintf("UpsertDocument DB Error: %v", err)) errorPage(c, http.StatusInternalServerError, fmt.Sprintf("UpsertDocument DB Error: %v", err))

84
api/ko-routes.go
View File

@ -19,7 +19,6 @@ import (
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"golang.org/x/exp/slices" "golang.org/x/exp/slices"
"reichard.io/bbank/database" "reichard.io/bbank/database"
"reichard.io/bbank/metadata"
) )
type activityItem struct { type activityItem struct {
@ -264,13 +263,13 @@ func (api *API) addActivities(c *gin.Context) {
// Add All Activity // Add All Activity
for _, item := range rActivity.Activity { for _, item := range rActivity.Activity {
if _, err := qtx.AddActivity(api.DB.Ctx, database.AddActivityParams{ if _, err := qtx.AddActivity(api.DB.Ctx, database.AddActivityParams{
UserID: rUser.(string), UserID: rUser.(string),
DocumentID: item.DocumentID, DocumentID: item.DocumentID,
DeviceID: rActivity.DeviceID, DeviceID: rActivity.DeviceID,
StartTime: time.Unix(int64(item.StartTime), 0).UTC().Format(time.RFC3339), StartTime: time.Unix(int64(item.StartTime), 0).UTC().Format(time.RFC3339),
Duration: int64(item.Duration), Duration: int64(item.Duration),
StartPercentage: float64(item.Page) / float64(item.Pages), Page: int64(item.Page),
EndPercentage: float64(item.Page+1) / float64(item.Pages), Pages: int64(item.Pages),
}); err != nil { }); err != nil {
log.Error("[addActivities] AddActivity DB Error:", err) log.Error("[addActivities] AddActivity DB Error:", err)
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Activity"}) c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Activity"})
@ -285,6 +284,14 @@ func (api *API) addActivities(c *gin.Context) {
return return
} }
// Update Temp Tables
go func() {
log.Info("[addActivities] Caching Temp Tables")
if err := api.DB.CacheTempTables(); err != nil {
log.Warn("[addActivities] CacheTempTables Failure: ", err)
}
}()
c.JSON(http.StatusOK, gin.H{ c.JSON(http.StatusOK, gin.H{
"added": len(rActivity.Activity), "added": len(rActivity.Activity),
}) })
@ -360,7 +367,7 @@ func (api *API) addDocuments(c *gin.Context) {
// Upsert Documents // Upsert Documents
for _, doc := range rNewDocs.Documents { for _, doc := range rNewDocs.Documents {
_, err := qtx.UpsertDocument(api.DB.Ctx, database.UpsertDocumentParams{ doc, err := qtx.UpsertDocument(api.DB.Ctx, database.UpsertDocumentParams{
ID: doc.ID, ID: doc.ID,
Title: api.sanitizeInput(doc.Title), Title: api.sanitizeInput(doc.Title),
Author: api.sanitizeInput(doc.Author), Author: api.sanitizeInput(doc.Author),
@ -374,6 +381,16 @@ func (api *API) addDocuments(c *gin.Context) {
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Document"}) c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Document"})
return return
} }
if _, err = qtx.UpdateDocumentSync(api.DB.Ctx, database.UpdateDocumentSyncParams{
ID: doc.ID,
Synced: true,
}); err != nil {
log.Error("[addDocuments] UpdateDocumentSync DB Error:", err)
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Document"})
return
}
} }
// Commit Transaction // Commit Transaction
@ -399,7 +416,7 @@ func (api *API) checkDocumentsSync(c *gin.Context) {
} }
// Upsert Device // Upsert Device
_, err := api.DB.Queries.UpsertDevice(api.DB.Ctx, database.UpsertDeviceParams{ device, err := api.DB.Queries.UpsertDevice(api.DB.Ctx, database.UpsertDeviceParams{
ID: rCheckDocs.DeviceID, ID: rCheckDocs.DeviceID,
UserID: rUser.(string), UserID: rUser.(string),
DeviceName: rCheckDocs.Device, DeviceName: rCheckDocs.Device,
@ -414,20 +431,22 @@ func (api *API) checkDocumentsSync(c *gin.Context) {
missingDocs := []database.Document{} missingDocs := []database.Document{}
deletedDocIDs := []string{} deletedDocIDs := []string{}
// Get Missing Documents if device.Sync == true {
missingDocs, err = api.DB.Queries.GetMissingDocuments(api.DB.Ctx, rCheckDocs.Have) // Get Missing Documents
if err != nil { missingDocs, err = api.DB.Queries.GetMissingDocuments(api.DB.Ctx, rCheckDocs.Have)
log.Error("[checkDocumentsSync] GetMissingDocuments DB Error", err) if err != nil {
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Request"}) log.Error("[checkDocumentsSync] GetMissingDocuments DB Error", err)
return c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Request"})
} return
}
// Get Deleted Documents // Get Deleted Documents
deletedDocIDs, err = api.DB.Queries.GetDeletedDocuments(api.DB.Ctx, rCheckDocs.Have) deletedDocIDs, err = api.DB.Queries.GetDeletedDocuments(api.DB.Ctx, rCheckDocs.Have)
if err != nil { if err != nil {
log.Error("[checkDocumentsSync] GetDeletedDocuments DB Error", err) log.Error("[checkDocumentsSync] GetDeletedDocuments DB Error", err)
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Request"}) c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Request"})
return return
}
} }
// Get Wanted Documents // Get Wanted Documents
@ -557,26 +576,27 @@ func (api *API) uploadExistingDocument(c *gin.Context) {
return return
} }
// Get Word Count
wordCount, err := metadata.GetWordCount(safePath)
if err != nil {
log.Error("[uploadExistingDocument] Word Count Failure:", err)
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "File Error"})
return
}
// Upsert Document // Upsert Document
if _, err = api.DB.Queries.UpsertDocument(api.DB.Ctx, database.UpsertDocumentParams{ if _, err = api.DB.Queries.UpsertDocument(api.DB.Ctx, database.UpsertDocumentParams{
ID: document.ID, ID: document.ID,
Md5: fileHash, Md5: fileHash,
Filepath: &fileName, Filepath: &fileName,
Words: &wordCount,
}); err != nil { }); err != nil {
log.Error("[uploadExistingDocument] UpsertDocument DB Error:", err) log.Error("[uploadExistingDocument] UpsertDocument DB Error:", err)
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Document Error"}) c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Document Error"})
return return
} }
// Update Document Sync Attribute
if _, err = api.DB.Queries.UpdateDocumentSync(api.DB.Ctx, database.UpdateDocumentSyncParams{
ID: document.ID,
Synced: true,
}); err != nil {
log.Error("[uploadExistingDocument] UpdateDocumentSync DB Error:", err)
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "Invalid Document"})
return
}
c.JSON(http.StatusOK, gin.H{ c.JSON(http.StatusOK, gin.H{
"status": "ok", "status": "ok",
}) })

2
api/opds-routes.go
View File

@ -56,7 +56,7 @@ func (api *API) opdsDocuments(c *gin.Context) {
fileType := splitFilepath[len(splitFilepath)-1] fileType := splitFilepath[len(splitFilepath)-1]
item := opds.Entry{ item := opds.Entry{
Title: fmt.Sprintf("[%3d%%] %s", int(doc.Percentage), *doc.Title), Title: fmt.Sprintf("[%3d%%] %s", int(doc.Percentage.(float64)), *doc.Title),
Author: []opds.Author{ Author: []opds.Author{
{ {
Name: *doc.Author, Name: *doc.Author,

4
database/manager.go
View File

@ -56,10 +56,6 @@ func NewMgr(c *config.Config) *DBManager {
return dbm return dbm
} }
func (dbm *DBManager) Shutdown() error {
return dbm.DB.Close()
}
func (dbm *DBManager) CacheTempTables() error { func (dbm *DBManager) CacheTempTables() error {
if _, err := dbm.DB.ExecContext(dbm.Ctx, tsql); err != nil { if _, err := dbm.DB.ExecContext(dbm.Ctx, tsql); err != nil {
return err return err

18
database/manager_test.go
View File

@ -122,13 +122,13 @@ func (dt *databaseTest) TestActivity() {
// Add Item // Add Item
activity, err := dt.dbm.Queries.AddActivity(dt.dbm.Ctx, AddActivityParams{ activity, err := dt.dbm.Queries.AddActivity(dt.dbm.Ctx, AddActivityParams{
DocumentID: documentID, DocumentID: documentID,
DeviceID: deviceID, DeviceID: deviceID,
UserID: userID, UserID: userID,
StartTime: d.UTC().Format(time.RFC3339), StartTime: d.UTC().Format(time.RFC3339),
Duration: 60, Duration: 60,
StartPercentage: float64(counter) / 100.0, Page: counter,
EndPercentage: float64(counter+1) / 100.0, Pages: 100,
}) })
// Validate No Error // Validate No Error
@ -143,7 +143,9 @@ func (dt *databaseTest) TestActivity() {
} }
// Initiate Cache // Initiate Cache
dt.dbm.CacheTempTables() if err := dt.dbm.CacheTempTables(); err != nil {
t.Fatalf(`Error: %v`, err)
}
// Validate Exists // Validate Exists
existsRows, err := dt.dbm.Queries.GetActivity(dt.dbm.Ctx, GetActivityParams{ existsRows, err := dt.dbm.Queries.GetActivity(dt.dbm.Ctx, GetActivityParams{

50
database/models.go
View File

@ -9,15 +9,14 @@ import (
) )
type Activity struct { type Activity struct {
ID int64 `json:"id"` UserID string `json:"user_id"`
UserID string `json:"user_id"` DocumentID string `json:"document_id"`
DocumentID string `json:"document_id"` DeviceID string `json:"device_id"`
DeviceID string `json:"device_id"` CreatedAt string `json:"created_at"`
StartTime string `json:"start_time"` StartTime string `json:"start_time"`
StartPercentage float64 `json:"start_percentage"` Page int64 `json:"page"`
EndPercentage float64 `json:"end_percentage"` Pages int64 `json:"pages"`
Duration int64 `json:"duration"` Duration int64 `json:"duration"`
CreatedAt string `json:"created_at"`
} }
type Device struct { type Device struct {
@ -64,8 +63,10 @@ type DocumentUserStatistic struct {
DocumentID string `json:"document_id"` DocumentID string `json:"document_id"`
UserID string `json:"user_id"` UserID string `json:"user_id"`
LastRead string `json:"last_read"` LastRead string `json:"last_read"`
Page int64 `json:"page"`
Pages int64 `json:"pages"`
TotalTimeSeconds int64 `json:"total_time_seconds"` TotalTimeSeconds int64 `json:"total_time_seconds"`
ReadPercentage float64 `json:"read_percentage"` ReadPages int64 `json:"read_pages"`
Percentage float64 `json:"percentage"` Percentage float64 `json:"percentage"`
WordsRead int64 `json:"words_read"` WordsRead int64 `json:"words_read"`
Wpm float64 `json:"wpm"` Wpm float64 `json:"wpm"`
@ -84,6 +85,18 @@ type Metadatum struct {
CreatedAt string `json:"created_at"` CreatedAt string `json:"created_at"`
} }
type RawActivity struct {
ID int64 `json:"id"`
UserID string `json:"user_id"`
DocumentID string `json:"document_id"`
DeviceID string `json:"device_id"`
StartTime string `json:"start_time"`
Page int64 `json:"page"`
Pages int64 `json:"pages"`
Duration int64 `json:"duration"`
CreatedAt string `json:"created_at"`
}
type User struct { type User struct {
ID string `json:"id"` ID string `json:"id"`
Pass *string `json:"-"` Pass *string `json:"-"`
@ -106,14 +119,27 @@ type UserStreak struct {
type ViewDocumentUserStatistic struct { type ViewDocumentUserStatistic struct {
DocumentID string `json:"document_id"` DocumentID string `json:"document_id"`
UserID string `json:"user_id"` UserID string `json:"user_id"`
LastRead interface{} `json:"last_read"` LastRead string `json:"last_read"`
Page int64 `json:"page"`
Pages int64 `json:"pages"`
TotalTimeSeconds sql.NullFloat64 `json:"total_time_seconds"` TotalTimeSeconds sql.NullFloat64 `json:"total_time_seconds"`
ReadPercentage sql.NullFloat64 `json:"read_percentage"` ReadPages int64 `json:"read_pages"`
Percentage float64 `json:"percentage"` Percentage float64 `json:"percentage"`
WordsRead interface{} `json:"words_read"` WordsRead interface{} `json:"words_read"`
Wpm int64 `json:"wpm"` Wpm int64 `json:"wpm"`
} }
type ViewRescaledActivity struct {
UserID string `json:"user_id"`
DocumentID string `json:"document_id"`
DeviceID string `json:"device_id"`
CreatedAt string `json:"created_at"`
StartTime string `json:"start_time"`
Page int64 `json:"page"`
Pages int64 `json:"pages"`
Duration int64 `json:"duration"`
}
type ViewUserStreak struct { type ViewUserStreak struct {
UserID string `json:"user_id"` UserID string `json:"user_id"`
Window string `json:"window"` Window string `json:"window"`

121
database/query.sql
View File

@ -1,12 +1,12 @@
-- name: AddActivity :one -- name: AddActivity :one
INSERT INTO activity ( INSERT INTO raw_activity (
user_id, user_id,
document_id, document_id,
device_id, device_id,
start_time, start_time,
duration, duration,
start_percentage, page,
end_percentage pages
) )
VALUES (?, ?, ?, ?, ?, ?, ?) VALUES (?, ?, ?, ?, ?, ?, ?)
RETURNING *; RETURNING *;
@ -43,7 +43,8 @@ WITH filtered_activity AS (
user_id, user_id,
start_time, start_time,
duration, duration,
ROUND(CAST(end_percentage - start_percentage AS REAL) * 100, 2) AS read_percentage page,
pages
FROM activity FROM activity
WHERE WHERE
activity.user_id = $user_id activity.user_id = $user_id
@ -64,7 +65,8 @@ SELECT
title, title,
author, author,
duration, duration,
read_percentage page,
pages
FROM filtered_activity AS activity FROM filtered_activity AS activity
LEFT JOIN documents ON documents.id = activity.document_id LEFT JOIN documents ON documents.id = activity.document_id
LEFT JOIN users ON users.id = activity.user_id; LEFT JOIN users ON users.id = activity.user_id;
@ -80,9 +82,9 @@ WITH RECURSIVE last_30_days AS (
), ),
filtered_activity AS ( filtered_activity AS (
SELECT SELECT
user_id, user_id,
start_time, start_time,
duration duration
FROM activity FROM activity
WHERE start_time > DATE('now', '-31 days') WHERE start_time > DATE('now', '-31 days')
AND activity.user_id = $user_id AND activity.user_id = $user_id
@ -140,6 +142,41 @@ ORDER BY devices.last_synced DESC;
SELECT * FROM documents SELECT * FROM documents
WHERE id = $document_id LIMIT 1; WHERE id = $document_id LIMIT 1;
-- name: GetDocumentDaysRead :one
WITH document_days AS (
SELECT DATE(start_time, time_offset) AS dates
FROM activity
JOIN users ON users.id = activity.user_id
WHERE document_id = $document_id
AND user_id = $user_id
GROUP BY dates
)
SELECT CAST(COUNT(*) AS INTEGER) AS days_read
FROM document_days;
-- name: GetDocumentReadStats :one
SELECT
COUNT(DISTINCT page) AS pages_read,
SUM(duration) AS total_time
FROM activity
WHERE document_id = $document_id
AND user_id = $user_id
AND start_time >= $start_time;
-- name: GetDocumentReadStatsCapped :one
WITH capped_stats AS (
SELECT MIN(SUM(duration), CAST($page_duration_cap AS INTEGER)) AS durations
FROM activity
WHERE document_id = $document_id
AND user_id = $user_id
AND start_time >= $start_time
GROUP BY page
)
SELECT
CAST(COUNT(*) AS INTEGER) AS pages_read,
CAST(SUM(durations) AS INTEGER) AS total_time
FROM capped_stats;
-- name: GetDocumentWithStats :one -- name: GetDocumentWithStats :one
SELECT SELECT
docs.id, docs.id,
@ -152,21 +189,23 @@ SELECT
docs.words, docs.words,
CAST(COALESCE(dus.wpm, 0.0) AS INTEGER) AS wpm, CAST(COALESCE(dus.wpm, 0.0) AS INTEGER) AS wpm,
COALESCE(dus.read_percentage, 0) AS read_percentage, COALESCE(dus.page, 0) AS page,
COALESCE(dus.pages, 0) AS pages,
COALESCE(dus.read_pages, 0) AS read_pages,
COALESCE(dus.total_time_seconds, 0) AS total_time_seconds, COALESCE(dus.total_time_seconds, 0) AS total_time_seconds,
STRFTIME('%Y-%m-%d %H:%M:%S', COALESCE(dus.last_read, "1970-01-01"), users.time_offset) STRFTIME('%Y-%m-%d %H:%M:%S', COALESCE(dus.last_read, "1970-01-01"), users.time_offset)
AS last_read, AS last_read,
ROUND(CAST(CASE CASE
WHEN dus.percentage > 97.0 THEN 100.0
WHEN dus.percentage IS NULL THEN 0.0 WHEN dus.percentage IS NULL THEN 0.0
WHEN (dus.percentage * 100.0) > 97.0 THEN 100.0 ELSE dus.percentage
ELSE dus.percentage * 100.0 END AS percentage,
END AS REAL), 2) AS percentage,
CAST(CASE CAST(CASE
WHEN dus.total_time_seconds IS NULL THEN 0.0 WHEN dus.total_time_seconds IS NULL THEN 0.0
ELSE ELSE
CAST(dus.total_time_seconds AS REAL) CAST(dus.total_time_seconds AS REAL)
/ (dus.read_percentage * 100.0) / CAST(dus.read_pages AS REAL)
END AS INTEGER) AS seconds_per_percent END AS INTEGER) AS seconds_per_page
FROM documents AS docs FROM documents AS docs
LEFT JOIN users ON users.id = $user_id LEFT JOIN users ON users.id = $user_id
LEFT JOIN LEFT JOIN
@ -194,24 +233,25 @@ SELECT
docs.words, docs.words,
CAST(COALESCE(dus.wpm, 0.0) AS INTEGER) AS wpm, CAST(COALESCE(dus.wpm, 0.0) AS INTEGER) AS wpm,
COALESCE(dus.read_percentage, 0) AS read_percentage, COALESCE(dus.page, 0) AS page,
COALESCE(dus.pages, 0) AS pages,
COALESCE(dus.read_pages, 0) AS read_pages,
COALESCE(dus.total_time_seconds, 0) AS total_time_seconds, COALESCE(dus.total_time_seconds, 0) AS total_time_seconds,
STRFTIME('%Y-%m-%d %H:%M:%S', COALESCE(dus.last_read, "1970-01-01"), users.time_offset) STRFTIME('%Y-%m-%d %H:%M:%S', COALESCE(dus.last_read, "1970-01-01"), users.time_offset)
AS last_read, AS last_read,
ROUND(CAST(CASE CASE
WHEN dus.percentage > 97.0 THEN 100.0
WHEN dus.percentage IS NULL THEN 0.0 WHEN dus.percentage IS NULL THEN 0.0
WHEN (dus.percentage * 100.0) > 97.0 THEN 100.0 ELSE dus.percentage
ELSE dus.percentage * 100.0 END AS percentage,
END AS REAL), 2) AS percentage,
CASE CASE
WHEN dus.total_time_seconds IS NULL THEN 0.0 WHEN dus.total_time_seconds IS NULL THEN 0.0
ELSE ELSE
ROUND( ROUND(
CAST(dus.total_time_seconds AS REAL) CAST(dus.total_time_seconds AS REAL)
/ (dus.read_percentage * 100.0) / CAST(dus.read_pages AS REAL)
) )
END AS seconds_per_percent END AS seconds_per_page
FROM documents AS docs FROM documents AS docs
LEFT JOIN users ON users.id = $user_id LEFT JOIN users ON users.id = $user_id
LEFT JOIN LEFT JOIN
@ -258,6 +298,20 @@ WHERE id = $user_id LIMIT 1;
SELECT * FROM user_streaks SELECT * FROM user_streaks
WHERE user_id = $user_id; WHERE user_id = $user_id;
-- name: GetUsers :many
SELECT * FROM users
WHERE
users.id = $user
OR ?1 IN (
SELECT id
FROM users
WHERE id = $user
AND admin = 1
)
ORDER BY created_at DESC
LIMIT $limit
OFFSET $offset;
-- name: GetWPMLeaderboard :many -- name: GetWPMLeaderboard :many
SELECT SELECT
user_id, user_id,
@ -274,18 +328,35 @@ ORDER BY wpm DESC;
SELECT SELECT
CAST(value AS TEXT) AS id, CAST(value AS TEXT) AS id,
CAST((documents.filepath IS NULL) AS BOOLEAN) AS want_file, CAST((documents.filepath IS NULL) AS BOOLEAN) AS want_file,
CAST((documents.id IS NULL) AS BOOLEAN) AS want_metadata CAST((IFNULL(documents.synced, false) != true) AS BOOLEAN) AS want_metadata
FROM json_each(?1) FROM json_each(?1)
LEFT JOIN documents LEFT JOIN documents
ON value = documents.id ON value = documents.id
WHERE ( WHERE (
documents.id IS NOT NULL documents.id IS NOT NULL
AND documents.deleted = false AND documents.deleted = false
AND documents.filepath IS NULL AND (
documents.synced = false
OR documents.filepath IS NULL
)
) )
OR (documents.id IS NULL) OR (documents.id IS NULL)
OR CAST($document_ids AS TEXT) != CAST($document_ids AS TEXT); OR CAST($document_ids AS TEXT) != CAST($document_ids AS TEXT);
-- name: UpdateDocumentDeleted :one
UPDATE documents
SET
deleted = $deleted
WHERE id = $id
RETURNING *;
-- name: UpdateDocumentSync :one
UPDATE documents
SET
synced = $synced
WHERE id = $id
RETURNING *;
-- name: UpdateProgress :one -- name: UpdateProgress :one
INSERT OR REPLACE INTO document_progress ( INSERT OR REPLACE INTO document_progress (
user_id, user_id,

397
database/query.sql.go
View File

@ -7,52 +7,53 @@ package database
import ( import (
"context" "context"
"database/sql"
"strings" "strings"
) )
const addActivity = `-- name: AddActivity :one const addActivity = `-- name: AddActivity :one
INSERT INTO activity ( INSERT INTO raw_activity (
user_id, user_id,
document_id, document_id,
device_id, device_id,
start_time, start_time,
duration, duration,
start_percentage, page,
end_percentage pages
) )
VALUES (?, ?, ?, ?, ?, ?, ?) VALUES (?, ?, ?, ?, ?, ?, ?)
RETURNING id, user_id, document_id, device_id, start_time, start_percentage, end_percentage, duration, created_at RETURNING id, user_id, document_id, device_id, start_time, page, pages, duration, created_at
` `
type AddActivityParams struct { type AddActivityParams struct {
UserID string `json:"user_id"` UserID string `json:"user_id"`
DocumentID string `json:"document_id"` DocumentID string `json:"document_id"`
DeviceID string `json:"device_id"` DeviceID string `json:"device_id"`
StartTime string `json:"start_time"` StartTime string `json:"start_time"`
Duration int64 `json:"duration"` Duration int64 `json:"duration"`
StartPercentage float64 `json:"start_percentage"` Page int64 `json:"page"`
EndPercentage float64 `json:"end_percentage"` Pages int64 `json:"pages"`
} }
func (q *Queries) AddActivity(ctx context.Context, arg AddActivityParams) (Activity, error) { func (q *Queries) AddActivity(ctx context.Context, arg AddActivityParams) (RawActivity, error) {
row := q.db.QueryRowContext(ctx, addActivity, row := q.db.QueryRowContext(ctx, addActivity,
arg.UserID, arg.UserID,
arg.DocumentID, arg.DocumentID,
arg.DeviceID, arg.DeviceID,
arg.StartTime, arg.StartTime,
arg.Duration, arg.Duration,
arg.StartPercentage, arg.Page,
arg.EndPercentage, arg.Pages,
) )
var i Activity var i RawActivity
err := row.Scan( err := row.Scan(
&i.ID, &i.ID,
&i.UserID, &i.UserID,
&i.DocumentID, &i.DocumentID,
&i.DeviceID, &i.DeviceID,
&i.StartTime, &i.StartTime,
&i.StartPercentage, &i.Page,
&i.EndPercentage, &i.Pages,
&i.Duration, &i.Duration,
&i.CreatedAt, &i.CreatedAt,
) )
@ -153,7 +154,8 @@ WITH filtered_activity AS (
user_id, user_id,
start_time, start_time,
duration, duration,
ROUND(CAST(end_percentage - start_percentage AS REAL) * 100, 2) AS read_percentage page,
pages
FROM activity FROM activity
WHERE WHERE
activity.user_id = ?1 activity.user_id = ?1
@ -174,7 +176,8 @@ SELECT
title, title,
author, author,
duration, duration,
read_percentage page,
pages
FROM filtered_activity AS activity FROM filtered_activity AS activity
LEFT JOIN documents ON documents.id = activity.document_id LEFT JOIN documents ON documents.id = activity.document_id
LEFT JOIN users ON users.id = activity.user_id LEFT JOIN users ON users.id = activity.user_id
@ -189,12 +192,13 @@ type GetActivityParams struct {
} }
type GetActivityRow struct { type GetActivityRow struct {
DocumentID string `json:"document_id"` DocumentID string `json:"document_id"`
StartTime string `json:"start_time"` StartTime string `json:"start_time"`
Title *string `json:"title"` Title *string `json:"title"`
Author *string `json:"author"` Author *string `json:"author"`
Duration int64 `json:"duration"` Duration int64 `json:"duration"`
ReadPercentage float64 `json:"read_percentage"` Page int64 `json:"page"`
Pages int64 `json:"pages"`
} }
func (q *Queries) GetActivity(ctx context.Context, arg GetActivityParams) ([]GetActivityRow, error) { func (q *Queries) GetActivity(ctx context.Context, arg GetActivityParams) ([]GetActivityRow, error) {
@ -218,7 +222,8 @@ func (q *Queries) GetActivity(ctx context.Context, arg GetActivityParams) ([]Get
&i.Title, &i.Title,
&i.Author, &i.Author,
&i.Duration, &i.Duration,
&i.ReadPercentage, &i.Page,
&i.Pages,
); err != nil { ); err != nil {
return nil, err return nil, err
} }
@ -244,9 +249,9 @@ WITH RECURSIVE last_30_days AS (
), ),
filtered_activity AS ( filtered_activity AS (
SELECT SELECT
user_id, user_id,
start_time, start_time,
duration duration
FROM activity FROM activity
WHERE start_time > DATE('now', '-31 days') WHERE start_time > DATE('now', '-31 days')
AND activity.user_id = ?1 AND activity.user_id = ?1
@ -460,6 +465,98 @@ func (q *Queries) GetDocument(ctx context.Context, documentID string) (Document,
return i, err return i, err
} }
const getDocumentDaysRead = `-- name: GetDocumentDaysRead :one
WITH document_days AS (
SELECT DATE(start_time, time_offset) AS dates
FROM activity
JOIN users ON users.id = activity.user_id
WHERE document_id = ?1
AND user_id = ?2
GROUP BY dates
)
SELECT CAST(COUNT(*) AS INTEGER) AS days_read
FROM document_days
`
type GetDocumentDaysReadParams struct {
DocumentID string `json:"document_id"`
UserID string `json:"user_id"`
}
func (q *Queries) GetDocumentDaysRead(ctx context.Context, arg GetDocumentDaysReadParams) (int64, error) {
row := q.db.QueryRowContext(ctx, getDocumentDaysRead, arg.DocumentID, arg.UserID)
var days_read int64
err := row.Scan(&days_read)
return days_read, err
}
const getDocumentReadStats = `-- name: GetDocumentReadStats :one
SELECT
COUNT(DISTINCT page) AS pages_read,
SUM(duration) AS total_time
FROM activity
WHERE document_id = ?1
AND user_id = ?2
AND start_time >= ?3
`
type GetDocumentReadStatsParams struct {
DocumentID string `json:"document_id"`
UserID string `json:"user_id"`
StartTime string `json:"start_time"`
}
type GetDocumentReadStatsRow struct {
PagesRead int64 `json:"pages_read"`
TotalTime sql.NullFloat64 `json:"total_time"`
}
func (q *Queries) GetDocumentReadStats(ctx context.Context, arg GetDocumentReadStatsParams) (GetDocumentReadStatsRow, error) {
row := q.db.QueryRowContext(ctx, getDocumentReadStats, arg.DocumentID, arg.UserID, arg.StartTime)
var i GetDocumentReadStatsRow
err := row.Scan(&i.PagesRead, &i.TotalTime)
return i, err
}
const getDocumentReadStatsCapped = `-- name: GetDocumentReadStatsCapped :one
WITH capped_stats AS (
SELECT MIN(SUM(duration), CAST(?1 AS INTEGER)) AS durations
FROM activity
WHERE document_id = ?2
AND user_id = ?3
AND start_time >= ?4
GROUP BY page
)
SELECT
CAST(COUNT(*) AS INTEGER) AS pages_read,
CAST(SUM(durations) AS INTEGER) AS total_time
FROM capped_stats
`
type GetDocumentReadStatsCappedParams struct {
PageDurationCap int64 `json:"page_duration_cap"`
DocumentID string `json:"document_id"`
UserID string `json:"user_id"`
StartTime string `json:"start_time"`
}
type GetDocumentReadStatsCappedRow struct {
PagesRead int64 `json:"pages_read"`
TotalTime int64 `json:"total_time"`
}
func (q *Queries) GetDocumentReadStatsCapped(ctx context.Context, arg GetDocumentReadStatsCappedParams) (GetDocumentReadStatsCappedRow, error) {
row := q.db.QueryRowContext(ctx, getDocumentReadStatsCapped,
arg.PageDurationCap,
arg.DocumentID,
arg.UserID,
arg.StartTime,
)
var i GetDocumentReadStatsCappedRow
err := row.Scan(&i.PagesRead, &i.TotalTime)
return i, err
}
const getDocumentWithStats = `-- name: GetDocumentWithStats :one const getDocumentWithStats = `-- name: GetDocumentWithStats :one
SELECT SELECT
docs.id, docs.id,
@ -472,21 +569,23 @@ SELECT
docs.words, docs.words,
CAST(COALESCE(dus.wpm, 0.0) AS INTEGER) AS wpm, CAST(COALESCE(dus.wpm, 0.0) AS INTEGER) AS wpm,
COALESCE(dus.read_percentage, 0) AS read_percentage, COALESCE(dus.page, 0) AS page,
COALESCE(dus.pages, 0) AS pages,
COALESCE(dus.read_pages, 0) AS read_pages,
COALESCE(dus.total_time_seconds, 0) AS total_time_seconds, COALESCE(dus.total_time_seconds, 0) AS total_time_seconds,
STRFTIME('%Y-%m-%d %H:%M:%S', COALESCE(dus.last_read, "1970-01-01"), users.time_offset) STRFTIME('%Y-%m-%d %H:%M:%S', COALESCE(dus.last_read, "1970-01-01"), users.time_offset)
AS last_read, AS last_read,
ROUND(CAST(CASE CASE
WHEN dus.percentage > 97.0 THEN 100.0
WHEN dus.percentage IS NULL THEN 0.0 WHEN dus.percentage IS NULL THEN 0.0
WHEN (dus.percentage * 100.0) > 97.0 THEN 100.0 ELSE dus.percentage
ELSE dus.percentage * 100.0 END AS percentage,
END AS REAL), 2) AS percentage,
CAST(CASE CAST(CASE
WHEN dus.total_time_seconds IS NULL THEN 0.0 WHEN dus.total_time_seconds IS NULL THEN 0.0
ELSE ELSE
CAST(dus.total_time_seconds AS REAL) CAST(dus.total_time_seconds AS REAL)
/ (dus.read_percentage * 100.0) / CAST(dus.read_pages AS REAL)
END AS INTEGER) AS seconds_per_percent END AS INTEGER) AS seconds_per_page
FROM documents AS docs FROM documents AS docs
LEFT JOIN users ON users.id = ?1 LEFT JOIN users ON users.id = ?1
LEFT JOIN LEFT JOIN
@ -503,20 +602,22 @@ type GetDocumentWithStatsParams struct {
} }
type GetDocumentWithStatsRow struct { type GetDocumentWithStatsRow struct {
ID string `json:"id"` ID string `json:"id"`
Title *string `json:"title"` Title *string `json:"title"`
Author *string `json:"author"` Author *string `json:"author"`
Description *string `json:"description"` Description *string `json:"description"`
Isbn10 *string `json:"isbn10"` Isbn10 *string `json:"isbn10"`
Isbn13 *string `json:"isbn13"` Isbn13 *string `json:"isbn13"`
Filepath *string `json:"filepath"` Filepath *string `json:"filepath"`
Words *int64 `json:"words"` Words *int64 `json:"words"`
Wpm int64 `json:"wpm"` Wpm int64 `json:"wpm"`
ReadPercentage float64 `json:"read_percentage"` Page int64 `json:"page"`
TotalTimeSeconds int64 `json:"total_time_seconds"` Pages int64 `json:"pages"`
LastRead interface{} `json:"last_read"` ReadPages int64 `json:"read_pages"`
Percentage float64 `json:"percentage"` TotalTimeSeconds int64 `json:"total_time_seconds"`
SecondsPerPercent int64 `json:"seconds_per_percent"` LastRead interface{} `json:"last_read"`
Percentage interface{} `json:"percentage"`
SecondsPerPage int64 `json:"seconds_per_page"`
} }
func (q *Queries) GetDocumentWithStats(ctx context.Context, arg GetDocumentWithStatsParams) (GetDocumentWithStatsRow, error) { func (q *Queries) GetDocumentWithStats(ctx context.Context, arg GetDocumentWithStatsParams) (GetDocumentWithStatsRow, error) {
@ -532,11 +633,13 @@ func (q *Queries) GetDocumentWithStats(ctx context.Context, arg GetDocumentWithS
&i.Filepath, &i.Filepath,
&i.Words, &i.Words,
&i.Wpm, &i.Wpm,
&i.ReadPercentage, &i.Page,
&i.Pages,
&i.ReadPages,
&i.TotalTimeSeconds, &i.TotalTimeSeconds,
&i.LastRead, &i.LastRead,
&i.Percentage, &i.Percentage,
&i.SecondsPerPercent, &i.SecondsPerPage,
) )
return i, err return i, err
} }
@ -608,24 +711,25 @@ SELECT
docs.words, docs.words,
CAST(COALESCE(dus.wpm, 0.0) AS INTEGER) AS wpm, CAST(COALESCE(dus.wpm, 0.0) AS INTEGER) AS wpm,
COALESCE(dus.read_percentage, 0) AS read_percentage, COALESCE(dus.page, 0) AS page,
COALESCE(dus.pages, 0) AS pages,
COALESCE(dus.read_pages, 0) AS read_pages,
COALESCE(dus.total_time_seconds, 0) AS total_time_seconds, COALESCE(dus.total_time_seconds, 0) AS total_time_seconds,
STRFTIME('%Y-%m-%d %H:%M:%S', COALESCE(dus.last_read, "1970-01-01"), users.time_offset) STRFTIME('%Y-%m-%d %H:%M:%S', COALESCE(dus.last_read, "1970-01-01"), users.time_offset)
AS last_read, AS last_read,
ROUND(CAST(CASE CASE
WHEN dus.percentage > 97.0 THEN 100.0
WHEN dus.percentage IS NULL THEN 0.0 WHEN dus.percentage IS NULL THEN 0.0
WHEN (dus.percentage * 100.0) > 97.0 THEN 100.0 ELSE dus.percentage
ELSE dus.percentage * 100.0 END AS percentage,
END AS REAL), 2) AS percentage,
CASE CASE
WHEN dus.total_time_seconds IS NULL THEN 0.0 WHEN dus.total_time_seconds IS NULL THEN 0.0
ELSE ELSE
ROUND( ROUND(
CAST(dus.total_time_seconds AS REAL) CAST(dus.total_time_seconds AS REAL)
/ (dus.read_percentage * 100.0) / CAST(dus.read_pages AS REAL)
) )
END AS seconds_per_percent END AS seconds_per_page
FROM documents AS docs FROM documents AS docs
LEFT JOIN users ON users.id = ?1 LEFT JOIN users ON users.id = ?1
LEFT JOIN LEFT JOIN
@ -644,20 +748,22 @@ type GetDocumentsWithStatsParams struct {
} }
type GetDocumentsWithStatsRow struct { type GetDocumentsWithStatsRow struct {
ID string `json:"id"` ID string `json:"id"`
Title *string `json:"title"` Title *string `json:"title"`
Author *string `json:"author"` Author *string `json:"author"`
Description *string `json:"description"` Description *string `json:"description"`
Isbn10 *string `json:"isbn10"` Isbn10 *string `json:"isbn10"`
Isbn13 *string `json:"isbn13"` Isbn13 *string `json:"isbn13"`
Filepath *string `json:"filepath"` Filepath *string `json:"filepath"`
Words *int64 `json:"words"` Words *int64 `json:"words"`
Wpm int64 `json:"wpm"` Wpm int64 `json:"wpm"`
ReadPercentage float64 `json:"read_percentage"` Page int64 `json:"page"`
TotalTimeSeconds int64 `json:"total_time_seconds"` Pages int64 `json:"pages"`
LastRead interface{} `json:"last_read"` ReadPages int64 `json:"read_pages"`
Percentage float64 `json:"percentage"` TotalTimeSeconds int64 `json:"total_time_seconds"`
SecondsPerPercent interface{} `json:"seconds_per_percent"` LastRead interface{} `json:"last_read"`
Percentage interface{} `json:"percentage"`
SecondsPerPage interface{} `json:"seconds_per_page"`
} }
func (q *Queries) GetDocumentsWithStats(ctx context.Context, arg GetDocumentsWithStatsParams) ([]GetDocumentsWithStatsRow, error) { func (q *Queries) GetDocumentsWithStats(ctx context.Context, arg GetDocumentsWithStatsParams) ([]GetDocumentsWithStatsRow, error) {
@ -679,11 +785,13 @@ func (q *Queries) GetDocumentsWithStats(ctx context.Context, arg GetDocumentsWit
&i.Filepath, &i.Filepath,
&i.Words, &i.Words,
&i.Wpm, &i.Wpm,
&i.ReadPercentage, &i.Page,
&i.Pages,
&i.ReadPages,
&i.TotalTimeSeconds, &i.TotalTimeSeconds,
&i.LastRead, &i.LastRead,
&i.Percentage, &i.Percentage,
&i.SecondsPerPercent, &i.SecondsPerPage,
); err != nil { ); err != nil {
return nil, err return nil, err
} }
@ -879,6 +987,56 @@ func (q *Queries) GetUserStreaks(ctx context.Context, userID string) ([]UserStre
return items, nil return items, nil
} }
const getUsers = `-- name: GetUsers :many
SELECT id, pass, admin, time_offset, created_at FROM users
WHERE
users.id = ?1
OR ?1 IN (
SELECT id
FROM users
WHERE id = ?1
AND admin = 1
)
ORDER BY created_at DESC
LIMIT ?3
OFFSET ?2
`
type GetUsersParams struct {
User string `json:"user"`
Offset int64 `json:"offset"`
Limit int64 `json:"limit"`
}
func (q *Queries) GetUsers(ctx context.Context, arg GetUsersParams) ([]User, error) {
rows, err := q.db.QueryContext(ctx, getUsers, arg.User, arg.Offset, arg.Limit)
if err != nil {
return nil, err
}
defer rows.Close()
var items []User
for rows.Next() {
var i User
if err := rows.Scan(
&i.ID,
&i.Pass,
&i.Admin,
&i.TimeOffset,
&i.CreatedAt,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Close(); err != nil {
return nil, err
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const getWPMLeaderboard = `-- name: GetWPMLeaderboard :many const getWPMLeaderboard = `-- name: GetWPMLeaderboard :many
SELECT SELECT
user_id, user_id,
@ -931,14 +1089,17 @@ const getWantedDocuments = `-- name: GetWantedDocuments :many
SELECT SELECT
CAST(value AS TEXT) AS id, CAST(value AS TEXT) AS id,
CAST((documents.filepath IS NULL) AS BOOLEAN) AS want_file, CAST((documents.filepath IS NULL) AS BOOLEAN) AS want_file,
CAST((documents.id IS NULL) AS BOOLEAN) AS want_metadata CAST((IFNULL(documents.synced, false) != true) AS BOOLEAN) AS want_metadata
FROM json_each(?1) FROM json_each(?1)
LEFT JOIN documents LEFT JOIN documents
ON value = documents.id ON value = documents.id
WHERE ( WHERE (
documents.id IS NOT NULL documents.id IS NOT NULL
AND documents.deleted = false AND documents.deleted = false
AND documents.filepath IS NULL AND (
documents.synced = false
OR documents.filepath IS NULL
)
) )
OR (documents.id IS NULL) OR (documents.id IS NULL)
OR CAST(?1 AS TEXT) != CAST(?1 AS TEXT) OR CAST(?1 AS TEXT) != CAST(?1 AS TEXT)
@ -973,6 +1134,86 @@ func (q *Queries) GetWantedDocuments(ctx context.Context, documentIds string) ([
return items, nil return items, nil
} }
const updateDocumentDeleted = `-- name: UpdateDocumentDeleted :one
UPDATE documents
SET
deleted = ?1
WHERE id = ?2
RETURNING id, md5, filepath, coverfile, title, author, series, series_index, lang, description, words, gbid, olid, isbn10, isbn13, synced, deleted, updated_at, created_at
`
type UpdateDocumentDeletedParams struct {
Deleted bool `json:"-"`
ID string `json:"id"`
}
func (q *Queries) UpdateDocumentDeleted(ctx context.Context, arg UpdateDocumentDeletedParams) (Document, error) {
row := q.db.QueryRowContext(ctx, updateDocumentDeleted, arg.Deleted, arg.ID)
var i Document
err := row.Scan(
&i.ID,
&i.Md5,
&i.Filepath,
&i.Coverfile,
&i.Title,
&i.Author,
&i.Series,
&i.SeriesIndex,
&i.Lang,
&i.Description,
&i.Words,
&i.Gbid,
&i.Olid,
&i.Isbn10,
&i.Isbn13,
&i.Synced,
&i.Deleted,
&i.UpdatedAt,
&i.CreatedAt,
)
return i, err
}
const updateDocumentSync = `-- name: UpdateDocumentSync :one
UPDATE documents
SET
synced = ?1
WHERE id = ?2
RETURNING id, md5, filepath, coverfile, title, author, series, series_index, lang, description, words, gbid, olid, isbn10, isbn13, synced, deleted, updated_at, created_at
`
type UpdateDocumentSyncParams struct {
Synced bool `json:"-"`
ID string `json:"id"`
}
func (q *Queries) UpdateDocumentSync(ctx context.Context, arg UpdateDocumentSyncParams) (Document, error) {
row := q.db.QueryRowContext(ctx, updateDocumentSync, arg.Synced, arg.ID)
var i Document
err := row.Scan(
&i.ID,
&i.Md5,
&i.Filepath,
&i.Coverfile,
&i.Title,
&i.Author,
&i.Series,
&i.SeriesIndex,
&i.Lang,
&i.Description,
&i.Words,
&i.Gbid,
&i.Olid,
&i.Isbn10,
&i.Isbn13,
&i.Synced,
&i.Deleted,
&i.UpdatedAt,
&i.CreatedAt,
)
return i, err
}
const updateProgress = `-- name: UpdateProgress :one const updateProgress = `-- name: UpdateProgress :one
INSERT OR REPLACE INTO document_progress ( INSERT OR REPLACE INTO document_progress (
user_id, user_id,

214
database/schema.sql
View File

@ -91,17 +91,16 @@ CREATE TABLE IF NOT EXISTS document_progress (
PRIMARY KEY (user_id, document_id, device_id) PRIMARY KEY (user_id, document_id, device_id)
); );
-- Read Activity -- Raw Read Activity
CREATE TABLE IF NOT EXISTS activity ( CREATE TABLE IF NOT EXISTS raw_activity (
id INTEGER PRIMARY KEY AUTOINCREMENT, id INTEGER PRIMARY KEY AUTOINCREMENT,
user_id TEXT NOT NULL, user_id TEXT NOT NULL,
document_id TEXT NOT NULL, document_id TEXT NOT NULL,
device_id TEXT NOT NULL, device_id TEXT NOT NULL,
start_time DATETIME NOT NULL, start_time DATETIME NOT NULL,
start_percentage REAL NOT NULL, page INTEGER NOT NULL,
end_percentage REAL NOT NULL, pages INTEGER NOT NULL,
duration INTEGER NOT NULL, duration INTEGER NOT NULL,
created_at DATETIME NOT NULL DEFAULT (STRFTIME('%Y-%m-%dT%H:%M:%SZ', 'now')), created_at DATETIME NOT NULL DEFAULT (STRFTIME('%Y-%m-%dT%H:%M:%SZ', 'now')),
@ -114,6 +113,19 @@ CREATE TABLE IF NOT EXISTS activity (
----------------------- Temporary Tables ---------------------- ----------------------- Temporary Tables ----------------------
--------------------------------------------------------------- ---------------------------------------------------------------
-- Temporary Activity Table (Cached from View)
CREATE TEMPORARY TABLE IF NOT EXISTS activity (
user_id TEXT NOT NULL,
document_id TEXT NOT NULL,
device_id TEXT NOT NULL,
created_at DATETIME NOT NULL,
start_time DATETIME NOT NULL,
page INTEGER NOT NULL,
pages INTEGER NOT NULL,
duration INTEGER NOT NULL
);
-- Temporary User Streaks Table (Cached from View) -- Temporary User Streaks Table (Cached from View)
CREATE TEMPORARY TABLE IF NOT EXISTS user_streaks ( CREATE TEMPORARY TABLE IF NOT EXISTS user_streaks (
user_id TEXT NOT NULL, user_id TEXT NOT NULL,
@ -132,8 +144,10 @@ CREATE TEMPORARY TABLE IF NOT EXISTS document_user_statistics (
document_id TEXT NOT NULL, document_id TEXT NOT NULL,
user_id TEXT NOT NULL, user_id TEXT NOT NULL,
last_read TEXT NOT NULL, last_read TEXT NOT NULL,
page INTEGER NOT NULL,
pages INTEGER NOT NULL,
total_time_seconds INTEGER NOT NULL, total_time_seconds INTEGER NOT NULL,
read_percentage REAL NOT NULL, read_pages INTEGER NOT NULL,
percentage REAL NOT NULL, percentage REAL NOT NULL,
words_read INTEGER NOT NULL, words_read INTEGER NOT NULL,
wpm REAL NOT NULL wpm REAL NOT NULL
@ -144,9 +158,9 @@ CREATE TEMPORARY TABLE IF NOT EXISTS document_user_statistics (
--------------------------- Indexes --------------------------- --------------------------- Indexes ---------------------------
--------------------------------------------------------------- ---------------------------------------------------------------
CREATE INDEX IF NOT EXISTS activity_start_time ON activity (start_time); CREATE INDEX IF NOT EXISTS temp.activity_start_time ON activity (start_time);
CREATE INDEX IF NOT EXISTS activity_user_id ON activity (user_id); CREATE INDEX IF NOT EXISTS temp.activity_user_id ON activity (user_id);
CREATE INDEX IF NOT EXISTS activity_user_id_document_id ON activity ( CREATE INDEX IF NOT EXISTS temp.activity_user_id_document_id ON activity (
user_id, user_id,
document_id document_id
); );
@ -155,6 +169,100 @@ CREATE INDEX IF NOT EXISTS activity_user_id_document_id ON activity (
---------------------------- Views ---------------------------- ---------------------------- Views ----------------------------
--------------------------------------------------------------- ---------------------------------------------------------------
--------------------------------
------- Rescaled Activity ------
--------------------------------
CREATE VIEW IF NOT EXISTS view_rescaled_activity AS
WITH RECURSIVE nums (idx) AS (
SELECT 1 AS idx
UNION ALL
SELECT idx + 1
FROM nums
LIMIT 1000
),
current_pages AS (
SELECT
document_id,
user_id,
pages
FROM raw_activity
GROUP BY document_id, user_id
HAVING MAX(start_time)
ORDER BY start_time DESC
),
intermediate AS (
SELECT
raw_activity.document_id,
raw_activity.device_id,
raw_activity.user_id,
raw_activity.created_at,
raw_activity.start_time,
raw_activity.duration,
raw_activity.page,
current_pages.pages,
-- Derive first page
((raw_activity.page - 1) * current_pages.pages) / raw_activity.pages
+ 1 AS first_page,
-- Derive last page
MAX(
((raw_activity.page - 1) * current_pages.pages)
/ raw_activity.pages
+ 1,
(raw_activity.page * current_pages.pages) / raw_activity.pages
) AS last_page
FROM raw_activity
INNER JOIN current_pages ON
current_pages.document_id = raw_activity.document_id
AND current_pages.user_id = raw_activity.user_id
),
num_limit AS (
SELECT * FROM nums
LIMIT (SELECT MAX(last_page - first_page + 1) FROM intermediate)
),
rescaled_raw AS (
SELECT
intermediate.document_id,
intermediate.device_id,
intermediate.user_id,
intermediate.created_at,
intermediate.start_time,
intermediate.last_page,
intermediate.pages,
intermediate.first_page + num_limit.idx - 1 AS page,
intermediate.duration / (
intermediate.last_page - intermediate.first_page + 1.0
) AS duration
FROM intermediate
LEFT JOIN num_limit ON
num_limit.idx <= (intermediate.last_page - intermediate.first_page + 1)
)
SELECT
user_id,
document_id,
device_id,
created_at,
start_time,
page,
pages,
-- Round up if last page (maintains total duration)
CAST(CASE
WHEN page = last_page AND duration != CAST(duration AS INTEGER)
THEN duration + 1
ELSE duration
END AS INTEGER) AS duration
FROM rescaled_raw;
-------------------------------- --------------------------------
--------- User Streaks --------- --------- User Streaks ---------
-------------------------------- --------------------------------
@ -171,7 +279,7 @@ WITH document_windows AS (
'weekday 0', '-7 day' 'weekday 0', '-7 day'
) AS weekly_read, ) AS weekly_read,
DATE(activity.start_time, users.time_offset) AS daily_read DATE(activity.start_time, users.time_offset) AS daily_read
FROM activity FROM raw_activity AS activity
LEFT JOIN users ON users.id = activity.user_id LEFT JOIN users ON users.id = activity.user_id
GROUP BY activity.user_id, weekly_read, daily_read GROUP BY activity.user_id, weekly_read, daily_read
), ),
@ -279,84 +387,38 @@ LEFT JOIN current_streak ON
CREATE VIEW IF NOT EXISTS view_document_user_statistics AS CREATE VIEW IF NOT EXISTS view_document_user_statistics AS
WITH intermediate_ga AS ( WITH true_progress AS (
SELECT SELECT
ga1.id AS row_id,
ga1.user_id,
ga1.document_id,
ga1.duration,
ga1.start_time,
ga1.start_percentage,
ga1.end_percentage,
-- Find Overlapping Events (Assign Unique ID)
(
SELECT MIN(id)
FROM activity AS ga2
WHERE
ga1.document_id = ga2.document_id
AND ga1.user_id = ga2.user_id
AND ga1.start_percentage <= ga2.end_percentage
AND ga1.end_percentage >= ga2.start_percentage
) AS group_leader
FROM activity AS ga1
),
grouped_activity AS (
SELECT
user_id,
document_id, document_id,
MAX(start_time) AS start_time,
MIN(start_percentage) AS start_percentage,
MAX(end_percentage) AS end_percentage,
MAX(end_percentage) - MIN(start_percentage) AS read_percentage,
SUM(duration) AS duration
FROM intermediate_ga
GROUP BY group_leader
),
current_progress AS (
SELECT
user_id, user_id,
document_id, start_time AS last_read,
COALESCE(( page,
SELECT percentage pages,
FROM document_progress AS dp SUM(duration) AS total_time_seconds,
WHERE
dp.user_id = iga.user_id -- Determine Read Pages
AND dp.document_id = iga.document_id COUNT(DISTINCT page) AS read_pages,
), end_percentage) AS percentage
FROM intermediate_ga AS iga -- Derive Percentage of Book
GROUP BY user_id, document_id ROUND(CAST(page AS REAL) / CAST(pages AS REAL) * 100, 2) AS percentage
FROM view_rescaled_activity
GROUP BY document_id, user_id
HAVING MAX(start_time) HAVING MAX(start_time)
) )
SELECT SELECT
ga.document_id, true_progress.*,
ga.user_id, (CAST(COALESCE(documents.words, 0.0) AS REAL) / pages * read_pages)
MAX(start_time) AS last_read,
SUM(duration) AS total_time_seconds,
SUM(read_percentage) AS read_percentage,
cp.percentage,
(CAST(COALESCE(d.words, 0.0) AS REAL) * SUM(read_percentage))
AS words_read, AS words_read,
(CAST(COALESCE(documents.words, 0.0) AS REAL) / pages * read_pages)
(CAST(COALESCE(d.words, 0.0) AS REAL) * SUM(read_percentage)) / (total_time_seconds / 60.0) AS wpm
/ (SUM(duration) / 60.0) AS wpm FROM true_progress
FROM grouped_activity AS ga INNER JOIN documents ON documents.id = true_progress.document_id
INNER JOIN
current_progress AS cp
ON ga.user_id = cp.user_id AND ga.document_id = cp.document_id
INNER JOIN
documents AS d
ON d.id = ga.document_id
GROUP BY ga.document_id, ga.user_id
ORDER BY wpm DESC; ORDER BY wpm DESC;
--------------------------------------------------------------- ---------------------------------------------------------------
------------------ Populate Temporary Tables ------------------ ------------------ Populate Temporary Tables ------------------
--------------------------------------------------------------- ---------------------------------------------------------------
INSERT INTO activity SELECT * FROM view_rescaled_activity;
INSERT INTO user_streaks SELECT * FROM view_user_streaks; INSERT INTO user_streaks SELECT * FROM view_user_streaks;
INSERT INTO document_user_statistics SELECT * FROM view_document_user_statistics; INSERT INTO document_user_statistics SELECT * FROM view_document_user_statistics;

2
database/update_temp_tables.sql
View File

@ -1,3 +1,5 @@
DELETE FROM activity;
INSERT INTO activity SELECT * FROM view_rescaled_activity;
DELETE FROM user_streaks; DELETE FROM user_streaks;
INSERT INTO user_streaks SELECT * FROM view_user_streaks; INSERT INTO user_streaks SELECT * FROM view_user_streaks;
DELETE FROM document_user_statistics; DELETE FROM document_user_statistics;

28
main.go
View File

@ -3,8 +3,6 @@ package main
import ( import (
"os" "os"
"os/signal" "os/signal"
"sync"
"syscall"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/urfave/cli/v2" "github.com/urfave/cli/v2"
@ -24,13 +22,13 @@ func main() {
log.SetFormatter(UTCFormatter{&log.TextFormatter{FullTimestamp: true}}) log.SetFormatter(UTCFormatter{&log.TextFormatter{FullTimestamp: true}})
app := &cli.App{ app := &cli.App{
Name: "Book Manager", Name: "Book Bank",
Usage: "A self hosted e-book progress tracker.", Usage: "A self hosted e-book progress tracker.",
Commands: []*cli.Command{ Commands: []*cli.Command{
{ {
Name: "serve", Name: "serve",
Aliases: []string{"s"}, Aliases: []string{"s"},
Usage: "Start Book Manager web server.", Usage: "Start Book Bank web server.",
Action: cmdServer, Action: cmdServer,
}, },
}, },
@ -42,23 +40,17 @@ func main() {
} }
func cmdServer(ctx *cli.Context) error { func cmdServer(ctx *cli.Context) error {
log.Info("Starting Book Manager Server") log.Info("Starting Book Bank Server")
// Create Channel
wg := sync.WaitGroup{}
done := make(chan struct{})
interrupt := make(chan os.Signal, 1)
signal.Notify(interrupt, os.Interrupt, syscall.SIGTERM)
// Start Server
server := server.NewServer() server := server.NewServer()
server.StartServer(&wg, done) server.StartServer()
// Wait & Close c := make(chan os.Signal, 1)
<-interrupt signal.Notify(c, os.Interrupt)
server.StopServer(&wg, done) <-c
// Stop Server log.Info("Stopping Server")
server.StopServer()
log.Info("Server Stopped")
os.Exit(0) os.Exit(0)
return nil return nil

66
server/server.go
View File

@ -5,7 +5,6 @@ import (
"net/http" "net/http"
"os" "os"
"path/filepath" "path/filepath"
"sync"
"time" "time"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
@ -30,72 +29,35 @@ func NewServer() *Server {
// Create Paths // Create Paths
docDir := filepath.Join(c.DataPath, "documents") docDir := filepath.Join(c.DataPath, "documents")
coversDir := filepath.Join(c.DataPath, "covers") coversDir := filepath.Join(c.DataPath, "covers")
os.Mkdir(docDir, os.ModePerm) _ = os.Mkdir(docDir, os.ModePerm)
os.Mkdir(coversDir, os.ModePerm) _ = os.Mkdir(coversDir, os.ModePerm)
return &Server{ return &Server{
API: api, API: api,
Config: c, Config: c,
Database: db, Database: db,
httpServer: &http.Server{
Handler: api.Router,
Addr: (":" + c.ListenPort),
},
} }
} }
func (s *Server) StartServer(wg *sync.WaitGroup, done <-chan struct{}) { func (s *Server) StartServer() {
ticker := time.NewTicker(15 * time.Minute) listenAddr := (":" + s.Config.ListenPort)
wg.Add(2) s.httpServer = &http.Server{
Handler: s.API.Router,
Addr: listenAddr,
}
go func() { go func() {
defer wg.Done()
err := s.httpServer.ListenAndServe() err := s.httpServer.ListenAndServe()
if err != nil && err != http.ErrServerClosed { if err != nil {
log.Error("Error Starting Server:", err) log.Error("Error starting server ", err)
}
}()
go func() {
defer wg.Done()
defer ticker.Stop()
s.RunScheduledTasks()
for {
select {
case <-ticker.C:
s.RunScheduledTasks()
case <-done:
log.Info("Stopping Task Runner...")
return
}
} }
}() }()
} }
func (s *Server) RunScheduledTasks() { func (s *Server) StopServer() {
log.Info("[RunScheduledTasks] Refreshing Temp Table Cache") ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
if err := s.API.DB.CacheTempTables(); err != nil {
log.Warn("[RunScheduledTasks] Refreshing Temp Table Cache Failure:", err)
}
log.Info("[RunScheduledTasks] Refreshing Temp Table Success")
}
func (s *Server) StopServer(wg *sync.WaitGroup, done chan<- struct{}) {
log.Info("Stopping HTTP Server...")
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel() defer cancel()
if err := s.httpServer.Shutdown(ctx); err != nil { s.httpServer.Shutdown(ctx)
log.Info("Shutting Error") s.API.DB.DB.Close()
}
s.API.DB.Shutdown()
close(done)
wg.Wait()
log.Info("Server Stopped")
} }

4
templates/activity.html
View File

@ -28,7 +28,7 @@
scope="col" scope="col"
class="p-3 font-normal text-left uppercase border-b border-gray-200 dark:border-gray-800" class="p-3 font-normal text-left uppercase border-b border-gray-200 dark:border-gray-800"
> >
Percent Page
</th> </th>
</tr> </tr>
</thead> </thead>
@ -51,7 +51,7 @@
<p>{{ $activity.Duration }}</p> <p>{{ $activity.Duration }}</p>
</td> </td>
<td class="p-3 border-b border-gray-200"> <td class="p-3 border-b border-gray-200">
<p>{{ $activity.ReadPercentage }}%</p> <p>{{ $activity.Page }} / {{ $activity.Pages }}</p>
</td> </td>
</tr> </tr>
{{end}} {{end}}

6
templates/document.html
View File

@ -326,9 +326,9 @@
<div class="absolute z-30 top-7 right-0 p-3 transition-all duration-200 bg-gray-200 rounded shadow-lg shadow-gray-500 dark:shadow-gray-900 dark:bg-gray-600"> <div class="absolute z-30 top-7 right-0 p-3 transition-all duration-200 bg-gray-200 rounded shadow-lg shadow-gray-500 dark:shadow-gray-900 dark:bg-gray-600">
<div class="text-xs flex"> <div class="text-xs flex">
<p class="text-gray-400 w-32">Seconds / Percent</p> <p class="text-gray-400 w-32">Seconds / Page</p>
<p class="font-medium dark:text-white"> <p class="font-medium dark:text-white">
{{ .Data.SecondsPerPercent }} {{ .Data.SecondsPerPage }}
</p> </p>
</div> </div>
<div class="text-xs flex"> <div class="text-xs flex">
@ -352,7 +352,7 @@
<div> <div>
<p class="text-gray-500">Progress</p> <p class="text-gray-500">Progress</p>
<p class="font-medium text-lg"> <p class="font-medium text-lg">
{{ .Data.Percentage }}% {{ .Data.Page }} / {{ .Data.Pages }} ({{ .Data.Percentage }}%)
</p> </p>
</div> </div>
<!-- <!--

2
templates/documents.html
View File

@ -37,7 +37,7 @@
<div> <div>
<p class="text-gray-400">Progress</p> <p class="text-gray-400">Progress</p>
<p class="font-medium"> <p class="font-medium">
{{ $doc.Percentage }}% {{ $doc.Page }} / {{ $doc.Pages }} ({{ $doc.Percentage }}%)
</p> </p>
</div> </div>
</div> </div>

2
utils/utils_test.go
View File

@ -2,7 +2,7 @@ package utils
import "testing" import "testing"
func TestCalculatePartialMD5(t *testing.T) { func TestCalculatePartialPD5(t *testing.T) {
partialMD5, err := CalculatePartialMD5("../_test_files/alice.epub") partialMD5, err := CalculatePartialMD5("../_test_files/alice.epub")
want := "386d1cb51fe4a72e5c9fdad5e059bad9" want := "386d1cb51fe4a72e5c9fdad5e059bad9"