evan/AnthoLume
1
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

[add] heavy query caching, [add] wpm leaderboard

Browse Source
This commit is contained in:
Evan Reichard
2023-10-03 16:47:38 -04:00
parent 5cd4e165b0
commit 4e1ee0022a
20 changed files with 1144 additions and 969 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
database/manager.go
View File

@@ -21,6 +21,9 @@ type DBManager struct {
//go:embed schema.sql
var ddl string
//go:embed update_temp_tables.sql
var tsql string
func NewMgr(c *config.Config) *DBManager {
// Create Manager
dbm := &DBManager{
@@ -44,22 +47,23 @@ func NewMgr(c *config.Config) *DBManager {
log.Fatal("Unsupported Database")
}
// Create Tables
if _, err := dbm.DB.ExecContext(dbm.Ctx, ddl); err != nil {
log.Fatal(err)
}
dbm.Queries = New(dbm.DB)
return dbm
}
func connectHookSQLite(conn *sqlite.SQLiteConn) error {
if err := conn.RegisterFunc("test_func", func() string {
return "FOOBAR"
}, false); err != nil {
log.Info("Error Registering Function")
func (dbm *DBManager) CacheTempTables() error {
if _, err := dbm.DB.ExecContext(dbm.Ctx, tsql); err != nil {
return err
}
return nil
}
func connectHookSQLite(conn *sqlite.SQLiteConn) error {
// Create Tables
log.Debug("Creating Schema")
if _, err := conn.Exec(ddl, nil); err != nil {
log.Warn("Create Schema Failure: ", err)
}
return nil
}

92
database/models.go
View File

@@ -5,27 +5,28 @@
package database
import (
"database/sql"
"time"
)
type Activity struct {
ID int64 `json:"id"`
UserID string `json:"user_id"`
DocumentID string `json:"document_id"`
DeviceID string `json:"device_id"`
CreatedAt time.Time `json:"created_at"`
StartTime time.Time `json:"start_time"`
Duration int64 `json:"duration"`
Page int64 `json:"page"`
Pages int64 `json:"pages"`
CreatedAt time.Time `json:"created_at"`
Duration int64 `json:"duration"`
}
type Device struct {
ID string `json:"id"`
UserID string `json:"user_id"`
DeviceName string `json:"device_name"`
CreatedAt string `json:"created_at"`
Sync bool `json:"sync"`
ID string `json:"id"`
UserID string `json:"user_id"`
DeviceName string `json:"device_name"`
LastSynced time.Time `json:"last_synced"`
CreatedAt string `json:"created_at"`
Sync bool `json:"sync"`
}
type Document struct {
@@ -50,14 +51,6 @@ type Document struct {
CreatedAt time.Time `json:"created_at"`
}
type DocumentDeviceSync struct {
UserID string `json:"user_id"`
DocumentID string `json:"document_id"`
DeviceID string `json:"device_id"`
LastSynced time.Time `json:"last_synced"`
Sync bool `json:"sync"`
}
type DocumentProgress struct {
UserID string `json:"user_id"`
DocumentID string `json:"document_id"`
@@ -67,6 +60,19 @@ type DocumentProgress struct {
CreatedAt time.Time `json:"created_at"`
}
type DocumentUserStatistic struct {
DocumentID string `json:"document_id"`
UserID string `json:"user_id"`
LastRead string `json:"last_read"`
Page int64 `json:"page"`
Pages int64 `json:"pages"`
TotalTimeSeconds int64 `json:"total_time_seconds"`
ReadPages int64 `json:"read_pages"`
Percentage float64 `json:"percentage"`
WordsRead int64 `json:"words_read"`
Wpm float64 `json:"wpm"`
}
type Metadatum struct {
ID int64 `json:"id"`
DocumentID string `json:"document_id"`
@@ -80,14 +86,16 @@ type Metadatum struct {
CreatedAt time.Time `json:"created_at"`
}
type RescaledActivity struct {
type RawActivity struct {
ID int64 `json:"id"`
UserID string `json:"user_id"`
DocumentID string `json:"document_id"`
DeviceID string `json:"device_id"`
UserID string `json:"user_id"`
StartTime time.Time `json:"start_time"`
Pages int64 `json:"pages"`
Page int64 `json:"page"`
Pages int64 `json:"pages"`
Duration int64 `json:"duration"`
CreatedAt time.Time `json:"created_at"`
}
type User struct {
@@ -97,3 +105,49 @@ type User struct {
TimeOffset *string `json:"time_offset"`
CreatedAt time.Time `json:"created_at"`
}
type UserStreak struct {
UserID string `json:"user_id"`
Window string `json:"window"`
MaxStreak int64 `json:"max_streak"`
MaxStreakStartDate string `json:"max_streak_start_date"`
MaxStreakEndDate string `json:"max_streak_end_date"`
CurrentStreak int64 `json:"current_streak"`
CurrentStreakStartDate string `json:"current_streak_start_date"`
CurrentStreakEndDate string `json:"current_streak_end_date"`
}
type ViewDocumentUserStatistic struct {
DocumentID string `json:"document_id"`
UserID string `json:"user_id"`
LastRead time.Time `json:"last_read"`
Page int64 `json:"page"`
Pages int64 `json:"pages"`
TotalTimeSeconds sql.NullFloat64 `json:"total_time_seconds"`
ReadPages int64 `json:"read_pages"`
Percentage float64 `json:"percentage"`
WordsRead interface{} `json:"words_read"`
Wpm int64 `json:"wpm"`
}
type ViewRescaledActivity struct {
UserID string `json:"user_id"`
DocumentID string `json:"document_id"`
DeviceID string `json:"device_id"`
CreatedAt time.Time `json:"created_at"`
StartTime time.Time `json:"start_time"`
Page int64 `json:"page"`
Pages int64 `json:"pages"`
Duration int64 `json:"duration"`
}
type ViewUserStreak struct {
UserID string `json:"user_id"`
Window string `json:"window"`
MaxStreak interface{} `json:"max_streak"`
MaxStreakStartDate interface{} `json:"max_streak_start_date"`
MaxStreakEndDate interface{} `json:"max_streak_end_date"`
CurrentStreak interface{} `json:"current_streak"`
CurrentStreakStartDate interface{} `json:"current_streak_start_date"`
CurrentStreakEndDate interface{} `json:"current_streak_end_date"`
}

758
database/query.sql
View File

@@ -1,3 +1,16 @@
-- name: AddActivity :one
INSERT INTO raw_activity (
user_id,
document_id,
device_id,
start_time,
duration,
page,
pages
)
VALUES (?, ?, ?, ?, ?, ?, ?)
RETURNING *;
-- name: AddMetadata :one
INSERT INTO metadata (
document_id,
@@ -17,10 +30,343 @@ INSERT INTO users (id, pass)
VALUES (?, ?)
ON CONFLICT DO NOTHING;
-- name: DeleteDocument :execrows
UPDATE documents
SET
deleted = 1
WHERE id = $id;
-- name: GetActivity :many
WITH filtered_activity AS (
SELECT
document_id,
user_id,
start_time,
duration,
page,
pages
FROM activity
WHERE
activity.user_id = $user_id
AND (
(
CAST($doc_filter AS BOOLEAN) = TRUE
AND document_id = $document_id
) OR $doc_filter = FALSE
)
ORDER BY start_time DESC
LIMIT $limit
OFFSET $offset
)
SELECT
document_id,
CAST(DATETIME(activity.start_time, users.time_offset) AS TEXT) AS start_time,
title,
author,
duration,
page,
pages
FROM filtered_activity AS activity
LEFT JOIN documents ON documents.id = activity.document_id
LEFT JOIN users ON users.id = activity.user_id;
-- name: GetDailyReadStats :many
WITH RECURSIVE last_30_days AS (
SELECT DATE('now', time_offset) AS date
FROM users WHERE users.id = $user_id
UNION ALL
SELECT DATE(date, '-1 days')
FROM last_30_days
LIMIT 30
),
filtered_activity AS (
SELECT
user_id,
start_time,
duration
FROM activity
WHERE start_time > DATE('now', '-31 days')
AND activity.user_id = $user_id
),
activity_days AS (
SELECT
SUM(duration) AS seconds_read,
DATE(start_time, time_offset) AS day
FROM filtered_activity AS activity
LEFT JOIN users ON users.id = activity.user_id
GROUP BY day
LIMIT 30
)
SELECT
CAST(date AS TEXT),
CAST(CASE
WHEN seconds_read IS NULL THEN 0
ELSE seconds_read / 60
END AS INTEGER) AS minutes_read
FROM last_30_days
LEFT JOIN activity_days ON activity_days.day == last_30_days.date
ORDER BY date DESC
LIMIT 30;
-- name: GetDatabaseInfo :one
SELECT
(SELECT COUNT(rowid) FROM activity WHERE activity.user_id = $user_id) AS activity_size,
(SELECT COUNT(rowid) FROM documents) AS documents_size,
(SELECT COUNT(rowid) FROM document_progress WHERE document_progress.user_id = $user_id) AS progress_size,
(SELECT COUNT(rowid) FROM devices WHERE devices.user_id = $user_id) AS devices_size
LIMIT 1;
-- name: GetDeletedDocuments :many
SELECT documents.id
FROM documents
WHERE
documents.deleted = true
AND documents.id IN (sqlc.slice('document_ids'));
-- name: GetDevice :one
SELECT * FROM devices
WHERE id = $device_id LIMIT 1;
-- name: GetDevices :many
SELECT
devices.device_name,
CAST(DATETIME(devices.created_at, users.time_offset) AS TEXT) AS created_at,
CAST(DATETIME(devices.last_synced, users.time_offset) AS TEXT) AS last_synced
FROM devices
JOIN users ON users.id = devices.user_id
WHERE users.id = $user_id;
-- name: GetDocument :one
SELECT * FROM documents
WHERE id = $document_id LIMIT 1;
-- name: GetDocumentDaysRead :one
WITH document_days AS (
SELECT DATE(start_time, time_offset) AS dates
FROM activity
JOIN users ON users.id = activity.user_id
WHERE document_id = $document_id
AND user_id = $user_id
GROUP BY dates
)
SELECT CAST(COUNT(*) AS INTEGER) AS days_read
FROM document_days;
-- name: GetDocumentReadStats :one
SELECT
COUNT(DISTINCT page) AS pages_read,
SUM(duration) AS total_time
FROM activity
WHERE document_id = $document_id
AND user_id = $user_id
AND start_time >= $start_time;
-- name: GetDocumentReadStatsCapped :one
WITH capped_stats AS (
SELECT MIN(SUM(duration), CAST($page_duration_cap AS INTEGER)) AS durations
FROM activity
WHERE document_id = $document_id
AND user_id = $user_id
AND start_time >= $start_time
GROUP BY page
)
SELECT
CAST(COUNT(*) AS INTEGER) AS pages_read,
CAST(SUM(durations) AS INTEGER) AS total_time
FROM capped_stats;
-- name: GetDocumentWithStats :one
SELECT
docs.id,
docs.title,
docs.author,
docs.description,
docs.isbn10,
docs.isbn13,
docs.filepath,
docs.words,
CAST(COALESCE(dus.wpm, 0.0) AS INTEGER) AS wpm,
COALESCE(dus.page, 0) AS page,
COALESCE(dus.pages, 0) AS pages,
COALESCE(dus.read_pages, 0) AS read_pages,
COALESCE(dus.total_time_seconds, 0) AS total_time_seconds,
DATETIME(COALESCE(dus.last_read, "1970-01-01"), users.time_offset)
AS last_read,
CASE
WHEN dus.percentage > 97.0 THEN 100.0
WHEN dus.percentage IS NULL THEN 0.0
ELSE dus.percentage
END AS percentage,
CAST(CASE
WHEN dus.total_time_seconds IS NULL THEN 0.0
ELSE
CAST(dus.total_time_seconds AS REAL)
/ CAST(dus.read_pages AS REAL)
END AS INTEGER) AS seconds_per_page
FROM documents AS docs
LEFT JOIN users ON users.id = $user_id
LEFT JOIN
document_user_statistics AS dus
ON dus.document_id = docs.id AND dus.user_id = $user_id
WHERE users.id = $user_id
AND docs.id = $document_id
LIMIT 1;
-- name: GetDocuments :many
SELECT * FROM documents
ORDER BY created_at DESC
LIMIT $limit
OFFSET $offset;
-- name: GetDocumentsWithStats :many
SELECT
docs.id,
docs.title,
docs.author,
docs.description,
docs.isbn10,
docs.isbn13,
docs.filepath,
docs.words,
CAST(COALESCE(dus.wpm, 0.0) AS INTEGER) AS wpm,
COALESCE(dus.page, 0) AS page,
COALESCE(dus.pages, 0) AS pages,
COALESCE(dus.read_pages, 0) AS read_pages,
COALESCE(dus.total_time_seconds, 0) AS total_time_seconds,
DATETIME(COALESCE(dus.last_read, "1970-01-01"), users.time_offset)
AS last_read,
CASE
WHEN dus.percentage > 97.0 THEN 100.0
WHEN dus.percentage IS NULL THEN 0.0
ELSE dus.percentage
END AS percentage,
CASE
WHEN dus.total_time_seconds IS NULL THEN 0.0
ELSE
ROUND(
CAST(dus.total_time_seconds AS REAL)
/ CAST(dus.read_pages AS REAL)
)
END AS seconds_per_page
FROM documents AS docs
LEFT JOIN users ON users.id = $user_id
LEFT JOIN
document_user_statistics AS dus
ON dus.document_id = docs.id AND dus.user_id = $user_id
WHERE docs.deleted = false
ORDER BY dus.last_read DESC, docs.created_at DESC
LIMIT $limit
OFFSET $offset;
-- name: GetLastActivity :one
SELECT start_time
FROM activity
WHERE device_id = $device_id
AND user_id = $user_id
ORDER BY start_time DESC LIMIT 1;
-- name: GetMissingDocuments :many
SELECT documents.* FROM documents
WHERE
documents.filepath IS NOT NULL
AND documents.deleted = false
AND documents.id NOT IN (sqlc.slice('document_ids'));
-- name: GetProgress :one
SELECT
document_progress.*,
devices.device_name
FROM document_progress
JOIN devices ON document_progress.device_id = devices.id
WHERE
document_progress.user_id = $user_id
AND document_progress.document_id = $document_id
ORDER BY
document_progress.created_at
DESC
LIMIT 1;
-- name: GetUser :one
SELECT * FROM users
WHERE id = $user_id LIMIT 1;
-- name: GetUserStreaks :many
SELECT * FROM user_streaks
WHERE user_id = $user_id;
-- name: GetUsers :many
SELECT * FROM users
WHERE
users.id = $user
OR ?1 IN (
SELECT id
FROM users
WHERE id = $user
AND admin = 1
)
ORDER BY created_at DESC
LIMIT $limit
OFFSET $offset;
-- name: GetWPMLeaderboard :many
SELECT
user_id,
CAST(SUM(words_read) AS INTEGER) AS total_words_read,
CAST(SUM(total_time_seconds) AS INTEGER) AS total_seconds,
ROUND(CAST(SUM(words_read) AS REAL) / (SUM(total_time_seconds) / 60.0), 2)
AS wpm
FROM document_user_statistics
WHERE words_read > 0
GROUP BY user_id
ORDER BY wpm DESC;
-- name: GetWantedDocuments :many
SELECT
CAST(value AS TEXT) AS id,
CAST((documents.filepath IS NULL) AS BOOLEAN) AS want_file,
CAST((IFNULL(documents.synced, false) != true) AS BOOLEAN) AS want_metadata
FROM json_each(?1)
LEFT JOIN documents
ON value = documents.id
WHERE (
documents.id IS NOT NULL
AND documents.deleted = false
AND (
documents.synced = false
OR documents.filepath IS NULL
)
)
OR (documents.id IS NULL)
OR CAST($document_ids AS TEXT) != CAST($document_ids AS TEXT);
-- name: UpdateDocumentDeleted :one
UPDATE documents
SET
deleted = $deleted
WHERE id = $id
RETURNING *;
-- name: UpdateDocumentSync :one
UPDATE documents
SET
synced = $synced
WHERE id = $id
RETURNING *;
-- name: UpdateProgress :one
INSERT OR REPLACE INTO document_progress (
user_id,
document_id,
device_id,
percentage,
progress
)
VALUES (?, ?, ?, ?, ?)
RETURNING *;
-- name: UpdateUser :one
UPDATE users
SET
@@ -29,6 +375,15 @@ SET
WHERE id = $user_id
RETURNING *;
-- name: UpsertDevice :one
INSERT INTO devices (id, user_id, last_synced, device_name)
VALUES (?, ?, ?, ?)
ON CONFLICT DO UPDATE
SET
device_name = COALESCE(excluded.device_name, device_name),
last_synced = COALESCE(excluded.last_synced, last_synced)
RETURNING *;
-- name: UpsertDocument :one
INSERT INTO documents (
id,
@@ -65,406 +420,3 @@ SET
isbn10 = COALESCE(excluded.isbn10, isbn10),
isbn13 = COALESCE(excluded.isbn13, isbn13)
RETURNING *;
-- name: DeleteDocument :execrows
UPDATE documents
SET
deleted = 1
WHERE id = $id;
-- name: UpdateDocumentSync :one
UPDATE documents
SET
synced = $synced
WHERE id = $id
RETURNING *;
-- name: UpdateDocumentDeleted :one
UPDATE documents
SET
deleted = $deleted
WHERE id = $id
RETURNING *;
-- name: GetDocument :one
SELECT * FROM documents
WHERE id = $document_id LIMIT 1;
-- name: UpsertDevice :one
INSERT INTO devices (id, user_id, device_name)
VALUES (?, ?, ?)
ON CONFLICT DO UPDATE
SET
device_name = COALESCE(excluded.device_name, device_name)
RETURNING *;
-- name: GetDevice :one
SELECT * FROM devices
WHERE id = $device_id LIMIT 1;
-- name: UpdateProgress :one
INSERT OR REPLACE INTO document_progress (
user_id,
document_id,
device_id,
percentage,
progress
)
VALUES (?, ?, ?, ?, ?)
RETURNING *;
-- name: GetProgress :one
SELECT
document_progress.*,
devices.device_name
FROM document_progress
JOIN devices ON document_progress.device_id = devices.id
WHERE
document_progress.user_id = $user_id
AND document_progress.document_id = $document_id
ORDER BY
document_progress.created_at
DESC
LIMIT 1;
-- name: GetLastActivity :one
SELECT start_time
FROM activity
WHERE device_id = $device_id
AND user_id = $user_id
ORDER BY start_time DESC LIMIT 1;
-- name: AddActivity :one
INSERT INTO activity (
user_id,
document_id,
device_id,
start_time,
duration,
page,
pages
)
VALUES (?, ?, ?, ?, ?, ?, ?)
RETURNING *;
-- name: GetMissingDocuments :many
SELECT documents.* FROM documents
WHERE
documents.filepath IS NOT NULL
AND documents.deleted = false
AND documents.id NOT IN (sqlc.slice('document_ids'));
-- name: GetWantedDocuments :many
SELECT
CAST(value AS TEXT) AS id,
CAST((documents.filepath IS NULL) AS BOOLEAN) AS want_file,
CAST((IFNULL(documents.synced, false) != true) AS BOOLEAN) AS want_metadata
FROM json_each(?1)
LEFT JOIN documents
ON value = documents.id
WHERE (
documents.id IS NOT NULL
AND documents.deleted = false
AND (
documents.synced = false
OR documents.filepath IS NULL
)
)
OR (documents.id IS NULL)
OR CAST($document_ids AS TEXT) != CAST($document_ids AS TEXT);
-- name: GetDeletedDocuments :many
SELECT documents.id
FROM documents
WHERE
documents.deleted = true
AND documents.id IN (sqlc.slice('document_ids'));
-- name: GetDocuments :many
SELECT * FROM documents
ORDER BY created_at DESC
LIMIT $limit
OFFSET $offset;
-- name: GetDocumentWithStats :one
WITH true_progress AS (
SELECT
start_time AS last_read,
SUM(duration) AS total_time_seconds,
document_id,
page,
pages,
-- Determine Read Pages
COUNT(DISTINCT page) AS read_pages,
-- Derive Percentage of Book
ROUND(CAST(page AS REAL) / CAST(pages AS REAL) * 100, 2) AS percentage
FROM rescaled_activity
WHERE user_id = $user_id
AND document_id = $document_id
GROUP BY document_id
HAVING MAX(start_time)
LIMIT 1
)
SELECT
documents.*,
CAST(IFNULL(page, 0) AS INTEGER) AS page,
CAST(IFNULL(pages, 0) AS INTEGER) AS pages,
CAST(IFNULL(total_time_seconds, 0) AS INTEGER) AS total_time_seconds,
CAST(DATETIME(IFNULL(last_read, "1970-01-01"), time_offset) AS TEXT) AS last_read,
CAST(IFNULL(read_pages, 0) AS INTEGER) AS read_pages,
-- Calculate Seconds / Page
-- 1. Calculate Total Time in Seconds (Sum Duration in Activity)
-- 2. Divide by Read Pages (Distinct Pages in Activity)
CAST(CASE
WHEN total_time_seconds IS NULL THEN 0.0
ELSE ROUND(CAST(total_time_seconds AS REAL) / CAST(read_pages AS REAL))
END AS INTEGER) AS seconds_per_page,
-- Arbitrarily >97% is Complete
CAST(CASE
WHEN percentage > 97.0 THEN 100.0
WHEN percentage IS NULL THEN 0.0
ELSE percentage
END AS REAL) AS percentage
FROM documents
LEFT JOIN true_progress ON true_progress.document_id = documents.id
LEFT JOIN users ON users.id = $user_id
WHERE documents.id = $document_id
ORDER BY true_progress.last_read DESC, documents.created_at DESC
LIMIT 1;
-- name: GetDocumentsWithStats :many
WITH true_progress AS (
SELECT
start_time AS last_read,
SUM(duration) AS total_time_seconds,
document_id,
page,
pages,
ROUND(CAST(page AS REAL) / CAST(pages AS REAL) * 100, 2) AS percentage
FROM activity
WHERE user_id = $user_id
GROUP BY document_id
HAVING MAX(start_time)
)
SELECT
documents.*,
CAST(IFNULL(page, 0) AS INTEGER) AS page,
CAST(IFNULL(pages, 0) AS INTEGER) AS pages,
CAST(IFNULL(total_time_seconds, 0) AS INTEGER) AS total_time_seconds,
CAST(DATETIME(IFNULL(last_read, "1970-01-01"), time_offset) AS TEXT) AS last_read,
CAST(CASE
WHEN percentage > 97.0 THEN 100.0
WHEN percentage IS NULL THEN 0.0
ELSE percentage
END AS REAL) AS percentage
FROM documents
LEFT JOIN true_progress ON true_progress.document_id = documents.id
LEFT JOIN users ON users.id = $user_id
WHERE documents.deleted == false
ORDER BY true_progress.last_read DESC, documents.created_at DESC
LIMIT $limit
OFFSET $offset;
-- name: GetUsers :many
SELECT * FROM users
WHERE
users.id = $user
OR ?1 IN (
SELECT id
FROM users
WHERE id = $user
AND admin = 1
)
ORDER BY created_at DESC
LIMIT $limit
OFFSET $offset;
-- name: GetActivity :many
SELECT
document_id,
CAST(DATETIME(activity.start_time, time_offset) AS TEXT) AS start_time,
title,
author,
duration,
page,
pages
FROM activity
LEFT JOIN documents ON documents.id = activity.document_id
LEFT JOIN users ON users.id = activity.user_id
WHERE
activity.user_id = $user_id
AND (
CAST($doc_filter AS BOOLEAN) = TRUE
AND document_id = $document_id
)
OR $doc_filter = FALSE
ORDER BY start_time DESC
LIMIT $limit
OFFSET $offset;
-- name: GetDevices :many
SELECT
devices.device_name,
CAST(DATETIME(devices.created_at, users.time_offset) AS TEXT) AS created_at,
CAST(DATETIME(MAX(activity.created_at), users.time_offset) AS TEXT) AS last_sync
FROM activity
JOIN devices ON devices.id = activity.device_id
JOIN users ON users.id = $user_id
WHERE devices.user_id = $user_id
GROUP BY activity.device_id;
-- name: GetDocumentReadStats :one
SELECT
COUNT(DISTINCT page) AS pages_read,
SUM(duration) AS total_time
FROM rescaled_activity
WHERE document_id = $document_id
AND user_id = $user_id
AND start_time >= $start_time;
-- name: GetDocumentReadStatsCapped :one
WITH capped_stats AS (
SELECT MIN(SUM(duration), CAST($page_duration_cap AS INTEGER)) AS durations
FROM rescaled_activity
WHERE document_id = $document_id
AND user_id = $user_id
AND start_time >= $start_time
GROUP BY page
)
SELECT
CAST(COUNT(*) AS INTEGER) AS pages_read,
CAST(SUM(durations) AS INTEGER) AS total_time
FROM capped_stats;
-- name: GetDocumentDaysRead :one
WITH document_days AS (
SELECT DATE(start_time, time_offset) AS dates
FROM activity
JOIN users ON users.id = activity.user_id
WHERE document_id = $document_id
AND user_id = $user_id
GROUP BY dates
)
SELECT CAST(COUNT(*) AS INTEGER) AS days_read
FROM document_days;
-- name: GetUserWindowStreaks :one
WITH document_windows AS (
SELECT
CASE
WHEN ?2 = "WEEK" THEN DATE(start_time, time_offset, 'weekday 0', '-7 day')
WHEN ?2 = "DAY" THEN DATE(start_time, time_offset)
END AS read_window,
time_offset
FROM activity
JOIN users ON users.id = activity.user_id
WHERE user_id = $user_id
AND CAST($window AS TEXT) = CAST($window AS TEXT)
GROUP BY read_window
),
partitions AS (
SELECT
document_windows.*,
row_number() OVER (
PARTITION BY 1 ORDER BY read_window DESC
) AS seqnum
FROM document_windows
),
streaks AS (
SELECT
COUNT(*) AS streak,
MIN(read_window) AS start_date,
MAX(read_window) AS end_date,
time_offset
FROM partitions
GROUP BY
CASE
WHEN ?2 = "DAY" THEN DATE(read_window, '+' || seqnum || ' day')
WHEN ?2 = "WEEK" THEN DATE(read_window, '+' || (seqnum * 7) || ' day')
END,
time_offset
ORDER BY end_date DESC
),
max_streak AS (
SELECT
MAX(streak) AS max_streak,
start_date AS max_streak_start_date,
end_date AS max_streak_end_date
FROM streaks
LIMIT 1
),
current_streak AS (
SELECT
streak AS current_streak,
start_date AS current_streak_start_date,
end_date AS current_streak_end_date
FROM streaks
WHERE CASE
WHEN ?2 = "WEEK" THEN
DATE('now', time_offset, 'weekday 0', '-14 day') = current_streak_end_date
OR DATE('now', time_offset, 'weekday 0', '-7 day') = current_streak_end_date
WHEN ?2 = "DAY" THEN
DATE('now', time_offset, '-1 day') = current_streak_end_date
OR DATE('now', time_offset) = current_streak_end_date
END
LIMIT 1
)
SELECT
CAST(IFNULL(max_streak, 0) AS INTEGER) AS max_streak,
CAST(IFNULL(max_streak_start_date, "N/A") AS TEXT) AS max_streak_start_date,
CAST(IFNULL(max_streak_end_date, "N/A") AS TEXT) AS max_streak_end_date,
IFNULL(current_streak, 0) AS current_streak,
CAST(IFNULL(current_streak_start_date, "N/A") AS TEXT) AS current_streak_start_date,
CAST(IFNULL(current_streak_end_date, "N/A") AS TEXT) AS current_streak_end_date
FROM max_streak
LEFT JOIN current_streak ON 1 = 1
LIMIT 1;
-- name: GetDatabaseInfo :one
SELECT
(SELECT COUNT(rowid) FROM activity WHERE activity.user_id = $user_id) AS activity_size,
(SELECT COUNT(rowid) FROM documents) AS documents_size,
(SELECT COUNT(rowid) FROM document_progress WHERE document_progress.user_id = $user_id) AS progress_size,
(SELECT COUNT(rowid) FROM devices WHERE devices.user_id = $user_id) AS devices_size
LIMIT 1;
-- name: GetDailyReadStats :many
WITH RECURSIVE last_30_days AS (
SELECT DATE('now', time_offset) AS date
FROM users WHERE users.id = $user_id
UNION ALL
SELECT DATE(date, '-1 days')
FROM last_30_days
LIMIT 30
),
activity_records AS (
SELECT
SUM(duration) AS seconds_read,
DATE(start_time, time_offset) AS day
FROM activity
LEFT JOIN users ON users.id = activity.user_id
WHERE user_id = $user_id
AND start_time > DATE('now', '-31 days')
GROUP BY day
ORDER BY day DESC
LIMIT 30
)
SELECT
CAST(date AS TEXT),
CAST(CASE
WHEN seconds_read IS NULL THEN 0
ELSE seconds_read / 60
END AS INTEGER) AS minutes_read
FROM last_30_days
LEFT JOIN activity_records ON activity_records.day == last_30_days.date
ORDER BY date DESC
LIMIT 30;

546
database/query.sql.go
View File

@@ -13,7 +13,7 @@ import (
)
const addActivity = `-- name: AddActivity :one
INSERT INTO activity (
INSERT INTO raw_activity (
user_id,
document_id,
device_id,
@@ -23,7 +23,7 @@ INSERT INTO activity (
pages
)
VALUES (?, ?, ?, ?, ?, ?, ?)
RETURNING id, user_id, document_id, device_id, start_time, duration, page, pages, created_at
RETURNING id, user_id, document_id, device_id, start_time, page, pages, duration, created_at
`
type AddActivityParams struct {
@@ -36,7 +36,7 @@ type AddActivityParams struct {
Pages int64 `json:"pages"`
}
func (q *Queries) AddActivity(ctx context.Context, arg AddActivityParams) (Activity, error) {
func (q *Queries) AddActivity(ctx context.Context, arg AddActivityParams) (RawActivity, error) {
row := q.db.QueryRowContext(ctx, addActivity,
arg.UserID,
arg.DocumentID,
@@ -46,16 +46,16 @@ func (q *Queries) AddActivity(ctx context.Context, arg AddActivityParams) (Activ
arg.Page,
arg.Pages,
)
var i Activity
var i RawActivity
err := row.Scan(
&i.ID,
&i.UserID,
&i.DocumentID,
&i.DeviceID,
&i.StartTime,
&i.Duration,
&i.Page,
&i.Pages,
&i.Duration,
&i.CreatedAt,
)
return i, err
@@ -149,27 +149,39 @@ func (q *Queries) DeleteDocument(ctx context.Context, id string) (int64, error)
}
const getActivity = `-- name: GetActivity :many
WITH filtered_activity AS (
SELECT
document_id,
user_id,
start_time,
duration,
page,
pages
FROM activity
WHERE
activity.user_id = ?1
AND (
(
CAST(?2 AS BOOLEAN) = TRUE
AND document_id = ?3
) OR ?2 = FALSE
)
ORDER BY start_time DESC
LIMIT ?5
OFFSET ?4
)
SELECT
document_id,
CAST(DATETIME(activity.start_time, time_offset) AS TEXT) AS start_time,
CAST(DATETIME(activity.start_time, users.time_offset) AS TEXT) AS start_time,
title,
author,
duration,
page,
pages
FROM activity
FROM filtered_activity AS activity
LEFT JOIN documents ON documents.id = activity.document_id
LEFT JOIN users ON users.id = activity.user_id
WHERE
activity.user_id = ?1
AND (
CAST(?2 AS BOOLEAN) = TRUE
AND document_id = ?3
)
OR ?2 = FALSE
ORDER BY start_time DESC
LIMIT ?5
OFFSET ?4
`
type GetActivityParams struct {
@@ -236,16 +248,22 @@ WITH RECURSIVE last_30_days AS (
FROM last_30_days
LIMIT 30
),
activity_records AS (
filtered_activity AS (
SELECT
user_id,
start_time,
duration
FROM activity
WHERE start_time > DATE('now', '-31 days')
AND activity.user_id = ?1
),
activity_days AS (
SELECT
SUM(duration) AS seconds_read,
DATE(start_time, time_offset) AS day
FROM activity
FROM filtered_activity AS activity
LEFT JOIN users ON users.id = activity.user_id
WHERE user_id = ?1
AND start_time > DATE('now', '-31 days')
GROUP BY day
ORDER BY day DESC
LIMIT 30
)
SELECT
@@ -255,7 +273,7 @@ SELECT
ELSE seconds_read / 60
END AS INTEGER) AS minutes_read
FROM last_30_days
LEFT JOIN activity_records ON activity_records.day == last_30_days.date
LEFT JOIN activity_days ON activity_days.day == last_30_days.date
ORDER BY date DESC
LIMIT 30
`
@@ -358,7 +376,7 @@ func (q *Queries) GetDeletedDocuments(ctx context.Context, documentIds []string)
}
const getDevice = `-- name: GetDevice :one
SELECT id, user_id, device_name, created_at, sync FROM devices
SELECT id, user_id, device_name, last_synced, created_at, sync FROM devices
WHERE id = ?1 LIMIT 1
`
@@ -369,6 +387,7 @@ func (q *Queries) GetDevice(ctx context.Context, deviceID string) (Device, error
&i.ID,
&i.UserID,
&i.DeviceName,
&i.LastSynced,
&i.CreatedAt,
&i.Sync,
)
@@ -379,18 +398,16 @@ const getDevices = `-- name: GetDevices :many
SELECT
devices.device_name,
CAST(DATETIME(devices.created_at, users.time_offset) AS TEXT) AS created_at,
CAST(DATETIME(MAX(activity.created_at), users.time_offset) AS TEXT) AS last_sync
FROM activity
JOIN devices ON devices.id = activity.device_id
JOIN users ON users.id = ?1
WHERE devices.user_id = ?1
GROUP BY activity.device_id
CAST(DATETIME(devices.last_synced, users.time_offset) AS TEXT) AS last_synced
FROM devices
JOIN users ON users.id = devices.user_id
WHERE users.id = ?1
`
type GetDevicesRow struct {
DeviceName string `json:"device_name"`
CreatedAt string `json:"created_at"`
LastSync string `json:"last_sync"`
LastSynced string `json:"last_synced"`
}
func (q *Queries) GetDevices(ctx context.Context, userID string) ([]GetDevicesRow, error) {
@@ -402,7 +419,7 @@ func (q *Queries) GetDevices(ctx context.Context, userID string) ([]GetDevicesRo
var items []GetDevicesRow
for rows.Next() {
var i GetDevicesRow
if err := rows.Scan(&i.DeviceName, &i.CreatedAt, &i.LastSync); err != nil {
if err := rows.Scan(&i.DeviceName, &i.CreatedAt, &i.LastSynced); err != nil {
return nil, err
}
items = append(items, i)
@@ -477,7 +494,7 @@ const getDocumentReadStats = `-- name: GetDocumentReadStats :one
SELECT
COUNT(DISTINCT page) AS pages_read,
SUM(duration) AS total_time
FROM rescaled_activity
FROM activity
WHERE document_id = ?1
AND user_id = ?2
AND start_time >= ?3
@@ -504,7 +521,7 @@ func (q *Queries) GetDocumentReadStats(ctx context.Context, arg GetDocumentReadS
const getDocumentReadStatsCapped = `-- name: GetDocumentReadStatsCapped :one
WITH capped_stats AS (
SELECT MIN(SUM(duration), CAST(?1 AS INTEGER)) AS durations
FROM rescaled_activity
FROM activity
WHERE document_id = ?2
AND user_id = ?3
AND start_time >= ?4
@@ -541,55 +558,41 @@ func (q *Queries) GetDocumentReadStatsCapped(ctx context.Context, arg GetDocumen
}
const getDocumentWithStats = `-- name: GetDocumentWithStats :one
WITH true_progress AS (
SELECT
start_time AS last_read,
SUM(duration) AS total_time_seconds,
document_id,
page,
pages,
-- Determine Read Pages
COUNT(DISTINCT page) AS read_pages,
-- Derive Percentage of Book
ROUND(CAST(page AS REAL) / CAST(pages AS REAL) * 100, 2) AS percentage
FROM rescaled_activity
WHERE user_id = ?1
AND document_id = ?2
GROUP BY document_id
HAVING MAX(start_time)
LIMIT 1
)
SELECT
documents.id, documents.md5, documents.filepath, documents.coverfile, documents.title, documents.author, documents.series, documents.series_index, documents.lang, documents.description, documents.words, documents.gbid, documents.olid, documents.isbn10, documents.isbn13, documents.synced, documents.deleted, documents.updated_at, documents.created_at,
docs.id,
docs.title,
docs.author,
docs.description,
docs.isbn10,
docs.isbn13,
docs.filepath,
docs.words,
CAST(IFNULL(page, 0) AS INTEGER) AS page,
CAST(IFNULL(pages, 0) AS INTEGER) AS pages,
CAST(IFNULL(total_time_seconds, 0) AS INTEGER) AS total_time_seconds,
CAST(DATETIME(IFNULL(last_read, "1970-01-01"), time_offset) AS TEXT) AS last_read,
CAST(IFNULL(read_pages, 0) AS INTEGER) AS read_pages,
-- Calculate Seconds / Page
-- 1. Calculate Total Time in Seconds (Sum Duration in Activity)
-- 2. Divide by Read Pages (Distinct Pages in Activity)
CAST(COALESCE(dus.wpm, 0.0) AS INTEGER) AS wpm,
COALESCE(dus.page, 0) AS page,
COALESCE(dus.pages, 0) AS pages,
COALESCE(dus.read_pages, 0) AS read_pages,
COALESCE(dus.total_time_seconds, 0) AS total_time_seconds,
DATETIME(COALESCE(dus.last_read, "1970-01-01"), users.time_offset)
AS last_read,
CASE
WHEN dus.percentage > 97.0 THEN 100.0
WHEN dus.percentage IS NULL THEN 0.0
ELSE dus.percentage
END AS percentage,
CAST(CASE
WHEN total_time_seconds IS NULL THEN 0.0
ELSE ROUND(CAST(total_time_seconds AS REAL) / CAST(read_pages AS REAL))
END AS INTEGER) AS seconds_per_page,
-- Arbitrarily >97% is Complete
CAST(CASE
WHEN percentage > 97.0 THEN 100.0
WHEN percentage IS NULL THEN 0.0
ELSE percentage
END AS REAL) AS percentage
FROM documents
LEFT JOIN true_progress ON true_progress.document_id = documents.id
WHEN dus.total_time_seconds IS NULL THEN 0.0
ELSE
CAST(dus.total_time_seconds AS REAL)
/ CAST(dus.read_pages AS REAL)
END AS INTEGER) AS seconds_per_page
FROM documents AS docs
LEFT JOIN users ON users.id = ?1
WHERE documents.id = ?2
ORDER BY true_progress.last_read DESC, documents.created_at DESC
LEFT JOIN
document_user_statistics AS dus
ON dus.document_id = docs.id AND dus.user_id = ?1
WHERE users.id = ?1
AND docs.id = ?2
LIMIT 1
`
@@ -599,32 +602,22 @@ type GetDocumentWithStatsParams struct {
}
type GetDocumentWithStatsRow struct {
ID string `json:"id"`
Md5 *string `json:"md5"`
Filepath *string `json:"filepath"`
Coverfile *string `json:"coverfile"`
Title *string `json:"title"`
Author *string `json:"author"`
Series *string `json:"series"`
SeriesIndex *int64 `json:"series_index"`
Lang *string `json:"lang"`
Description *string `json:"description"`
Words *int64 `json:"words"`
Gbid *string `json:"gbid"`
Olid *string `json:"-"`
Isbn10 *string `json:"isbn10"`
Isbn13 *string `json:"isbn13"`
Synced bool `json:"-"`
Deleted bool `json:"-"`
UpdatedAt time.Time `json:"updated_at"`
CreatedAt time.Time `json:"created_at"`
Page int64 `json:"page"`
Pages int64 `json:"pages"`
TotalTimeSeconds int64 `json:"total_time_seconds"`
LastRead string `json:"last_read"`
ReadPages int64 `json:"read_pages"`
SecondsPerPage int64 `json:"seconds_per_page"`
Percentage float64 `json:"percentage"`
ID string `json:"id"`
Title *string `json:"title"`
Author *string `json:"author"`
Description *string `json:"description"`
Isbn10 *string `json:"isbn10"`
Isbn13 *string `json:"isbn13"`
Filepath *string `json:"filepath"`
Words *int64 `json:"words"`
Wpm int64 `json:"wpm"`
Page int64 `json:"page"`
Pages int64 `json:"pages"`
ReadPages int64 `json:"read_pages"`
TotalTimeSeconds int64 `json:"total_time_seconds"`
LastRead interface{} `json:"last_read"`
Percentage interface{} `json:"percentage"`
SecondsPerPage int64 `json:"seconds_per_page"`
}
func (q *Queries) GetDocumentWithStats(ctx context.Context, arg GetDocumentWithStatsParams) (GetDocumentWithStatsRow, error) {
@@ -632,31 +625,21 @@ func (q *Queries) GetDocumentWithStats(ctx context.Context, arg GetDocumentWithS
var i GetDocumentWithStatsRow
err := row.Scan(
&i.ID,
&i.Md5,
&i.Filepath,
&i.Coverfile,
&i.Title,
&i.Author,
&i.Series,
&i.SeriesIndex,
&i.Lang,
&i.Description,
&i.Words,
&i.Gbid,
&i.Olid,
&i.Isbn10,
&i.Isbn13,
&i.Synced,
&i.Deleted,
&i.UpdatedAt,
&i.CreatedAt,
&i.Filepath,
&i.Words,
&i.Wpm,
&i.Page,
&i.Pages,
&i.ReadPages,
&i.TotalTimeSeconds,
&i.LastRead,
&i.ReadPages,
&i.SecondsPerPage,
&i.Percentage,
&i.SecondsPerPage,
)
return i, err
}
@@ -717,38 +700,43 @@ func (q *Queries) GetDocuments(ctx context.Context, arg GetDocumentsParams) ([]D
}
const getDocumentsWithStats = `-- name: GetDocumentsWithStats :many
WITH true_progress AS (
SELECT
start_time AS last_read,
SUM(duration) AS total_time_seconds,
document_id,
page,
pages,
ROUND(CAST(page AS REAL) / CAST(pages AS REAL) * 100, 2) AS percentage
FROM activity
WHERE user_id = ?1
GROUP BY document_id
HAVING MAX(start_time)
)
SELECT
documents.id, documents.md5, documents.filepath, documents.coverfile, documents.title, documents.author, documents.series, documents.series_index, documents.lang, documents.description, documents.words, documents.gbid, documents.olid, documents.isbn10, documents.isbn13, documents.synced, documents.deleted, documents.updated_at, documents.created_at,
docs.id,
docs.title,
docs.author,
docs.description,
docs.isbn10,
docs.isbn13,
docs.filepath,
docs.words,
CAST(IFNULL(page, 0) AS INTEGER) AS page,
CAST(IFNULL(pages, 0) AS INTEGER) AS pages,
CAST(IFNULL(total_time_seconds, 0) AS INTEGER) AS total_time_seconds,
CAST(DATETIME(IFNULL(last_read, "1970-01-01"), time_offset) AS TEXT) AS last_read,
CAST(CASE
WHEN percentage > 97.0 THEN 100.0
WHEN percentage IS NULL THEN 0.0
ELSE percentage
END AS REAL) AS percentage
FROM documents
LEFT JOIN true_progress ON true_progress.document_id = documents.id
CAST(COALESCE(dus.wpm, 0.0) AS INTEGER) AS wpm,
COALESCE(dus.page, 0) AS page,
COALESCE(dus.pages, 0) AS pages,
COALESCE(dus.read_pages, 0) AS read_pages,
COALESCE(dus.total_time_seconds, 0) AS total_time_seconds,
DATETIME(COALESCE(dus.last_read, "1970-01-01"), users.time_offset)
AS last_read,
CASE
WHEN dus.percentage > 97.0 THEN 100.0
WHEN dus.percentage IS NULL THEN 0.0
ELSE dus.percentage
END AS percentage,
CASE
WHEN dus.total_time_seconds IS NULL THEN 0.0
ELSE
ROUND(
CAST(dus.total_time_seconds AS REAL)
/ CAST(dus.read_pages AS REAL)
)
END AS seconds_per_page
FROM documents AS docs
LEFT JOIN users ON users.id = ?1
WHERE documents.deleted == false
ORDER BY true_progress.last_read DESC, documents.created_at DESC
LEFT JOIN
document_user_statistics AS dus
ON dus.document_id = docs.id AND dus.user_id = ?1
WHERE docs.deleted = false
ORDER BY dus.last_read DESC, docs.created_at DESC
LIMIT ?3
OFFSET ?2
`
@@ -760,30 +748,22 @@ type GetDocumentsWithStatsParams struct {
}
type GetDocumentsWithStatsRow struct {
ID string `json:"id"`
Md5 *string `json:"md5"`
Filepath *string `json:"filepath"`
Coverfile *string `json:"coverfile"`
Title *string `json:"title"`
Author *string `json:"author"`
Series *string `json:"series"`
SeriesIndex *int64 `json:"series_index"`
Lang *string `json:"lang"`
Description *string `json:"description"`
Words *int64 `json:"words"`
Gbid *string `json:"gbid"`
Olid *string `json:"-"`
Isbn10 *string `json:"isbn10"`
Isbn13 *string `json:"isbn13"`
Synced bool `json:"-"`
Deleted bool `json:"-"`
UpdatedAt time.Time `json:"updated_at"`
CreatedAt time.Time `json:"created_at"`
Page int64 `json:"page"`
Pages int64 `json:"pages"`
TotalTimeSeconds int64 `json:"total_time_seconds"`
LastRead string `json:"last_read"`
Percentage float64 `json:"percentage"`
ID string `json:"id"`
Title *string `json:"title"`
Author *string `json:"author"`
Description *string `json:"description"`
Isbn10 *string `json:"isbn10"`
Isbn13 *string `json:"isbn13"`
Filepath *string `json:"filepath"`
Words *int64 `json:"words"`
Wpm int64 `json:"wpm"`
Page int64 `json:"page"`
Pages int64 `json:"pages"`
ReadPages int64 `json:"read_pages"`
TotalTimeSeconds int64 `json:"total_time_seconds"`
LastRead interface{} `json:"last_read"`
Percentage interface{} `json:"percentage"`
SecondsPerPage interface{} `json:"seconds_per_page"`
}
func (q *Queries) GetDocumentsWithStats(ctx context.Context, arg GetDocumentsWithStatsParams) ([]GetDocumentsWithStatsRow, error) {
@@ -797,29 +777,21 @@ func (q *Queries) GetDocumentsWithStats(ctx context.Context, arg GetDocumentsWit
var i GetDocumentsWithStatsRow
if err := rows.Scan(
&i.ID,
&i.Md5,
&i.Filepath,
&i.Coverfile,
&i.Title,
&i.Author,
&i.Series,
&i.SeriesIndex,
&i.Lang,
&i.Description,
&i.Words,
&i.Gbid,
&i.Olid,
&i.Isbn10,
&i.Isbn13,
&i.Synced,
&i.Deleted,
&i.UpdatedAt,
&i.CreatedAt,
&i.Filepath,
&i.Words,
&i.Wpm,
&i.Page,
&i.Pages,
&i.ReadPages,
&i.TotalTimeSeconds,
&i.LastRead,
&i.Percentage,
&i.SecondsPerPage,
); err != nil {
return nil, err
}
@@ -978,105 +950,41 @@ func (q *Queries) GetUser(ctx context.Context, userID string) (User, error) {
return i, err
}
const getUserWindowStreaks = `-- name: GetUserWindowStreaks :one
WITH document_windows AS (
SELECT
CASE
WHEN ?2 = "WEEK" THEN DATE(start_time, time_offset, 'weekday 0', '-7 day')
WHEN ?2 = "DAY" THEN DATE(start_time, time_offset)
END AS read_window,
time_offset
FROM activity
JOIN users ON users.id = activity.user_id
WHERE user_id = ?1
AND CAST(?2 AS TEXT) = CAST(?2 AS TEXT)
GROUP BY read_window
),
partitions AS (
SELECT
document_windows.read_window, document_windows.time_offset,
row_number() OVER (
PARTITION BY 1 ORDER BY read_window DESC
) AS seqnum
FROM document_windows
),
streaks AS (
SELECT
COUNT(*) AS streak,
MIN(read_window) AS start_date,
MAX(read_window) AS end_date,
time_offset
FROM partitions
GROUP BY
CASE
WHEN ?2 = "DAY" THEN DATE(read_window, '+' || seqnum || ' day')
WHEN ?2 = "WEEK" THEN DATE(read_window, '+' || (seqnum * 7) || ' day')
END,
time_offset
ORDER BY end_date DESC
),
max_streak AS (
SELECT
MAX(streak) AS max_streak,
start_date AS max_streak_start_date,
end_date AS max_streak_end_date
FROM streaks
LIMIT 1
),
current_streak AS (
SELECT
streak AS current_streak,
start_date AS current_streak_start_date,
end_date AS current_streak_end_date
FROM streaks
WHERE CASE
WHEN ?2 = "WEEK" THEN
DATE('now', time_offset, 'weekday 0', '-14 day') = current_streak_end_date
OR DATE('now', time_offset, 'weekday 0', '-7 day') = current_streak_end_date
WHEN ?2 = "DAY" THEN
DATE('now', time_offset, '-1 day') = current_streak_end_date
OR DATE('now', time_offset) = current_streak_end_date
END
LIMIT 1
)
SELECT
CAST(IFNULL(max_streak, 0) AS INTEGER) AS max_streak,
CAST(IFNULL(max_streak_start_date, "N/A") AS TEXT) AS max_streak_start_date,
CAST(IFNULL(max_streak_end_date, "N/A") AS TEXT) AS max_streak_end_date,
IFNULL(current_streak, 0) AS current_streak,
CAST(IFNULL(current_streak_start_date, "N/A") AS TEXT) AS current_streak_start_date,
CAST(IFNULL(current_streak_end_date, "N/A") AS TEXT) AS current_streak_end_date
FROM max_streak
LEFT JOIN current_streak ON 1 = 1
LIMIT 1
const getUserStreaks = `-- name: GetUserStreaks :many
SELECT user_id, "window", max_streak, max_streak_start_date, max_streak_end_date, current_streak, current_streak_start_date, current_streak_end_date FROM user_streaks
WHERE user_id = ?1
`
type GetUserWindowStreaksParams struct {
UserID string `json:"user_id"`
Window string `json:"window"`
}
type GetUserWindowStreaksRow struct {
MaxStreak int64 `json:"max_streak"`
MaxStreakStartDate string `json:"max_streak_start_date"`
MaxStreakEndDate string `json:"max_streak_end_date"`
CurrentStreak interface{} `json:"current_streak"`
CurrentStreakStartDate string `json:"current_streak_start_date"`
CurrentStreakEndDate string `json:"current_streak_end_date"`
}
func (q *Queries) GetUserWindowStreaks(ctx context.Context, arg GetUserWindowStreaksParams) (GetUserWindowStreaksRow, error) {
row := q.db.QueryRowContext(ctx, getUserWindowStreaks, arg.UserID, arg.Window)
var i GetUserWindowStreaksRow
err := row.Scan(
&i.MaxStreak,
&i.MaxStreakStartDate,
&i.MaxStreakEndDate,
&i.CurrentStreak,
&i.CurrentStreakStartDate,
&i.CurrentStreakEndDate,
)
return i, err
func (q *Queries) GetUserStreaks(ctx context.Context, userID string) ([]UserStreak, error) {
rows, err := q.db.QueryContext(ctx, getUserStreaks, userID)
if err != nil {
return nil, err
}
defer rows.Close()
var items []UserStreak
for rows.Next() {
var i UserStreak
if err := rows.Scan(
&i.UserID,
&i.Window,
&i.MaxStreak,
&i.MaxStreakStartDate,
&i.MaxStreakEndDate,
&i.CurrentStreak,
&i.CurrentStreakStartDate,
&i.CurrentStreakEndDate,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Close(); err != nil {
return nil, err
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const getUsers = `-- name: GetUsers :many
@@ -1129,6 +1037,54 @@ func (q *Queries) GetUsers(ctx context.Context, arg GetUsersParams) ([]User, err
return items, nil
}
const getWPMLeaderboard = `-- name: GetWPMLeaderboard :many
SELECT
user_id,
CAST(SUM(words_read) AS INTEGER) AS total_words_read,
CAST(SUM(total_time_seconds) AS INTEGER) AS total_seconds,
ROUND(CAST(SUM(words_read) AS REAL) / (SUM(total_time_seconds) / 60.0), 2)
AS wpm
FROM document_user_statistics
WHERE words_read > 0
GROUP BY user_id
ORDER BY wpm DESC
`
type GetWPMLeaderboardRow struct {
UserID string `json:"user_id"`
TotalWordsRead int64 `json:"total_words_read"`
TotalSeconds int64 `json:"total_seconds"`
Wpm float64 `json:"wpm"`
}
func (q *Queries) GetWPMLeaderboard(ctx context.Context) ([]GetWPMLeaderboardRow, error) {
rows, err := q.db.QueryContext(ctx, getWPMLeaderboard)
if err != nil {
return nil, err
}
defer rows.Close()
var items []GetWPMLeaderboardRow
for rows.Next() {
var i GetWPMLeaderboardRow
if err := rows.Scan(
&i.UserID,
&i.TotalWordsRead,
&i.TotalSeconds,
&i.Wpm,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Close(); err != nil {
return nil, err
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const getWantedDocuments = `-- name: GetWantedDocuments :many
SELECT
CAST(value AS TEXT) AS id,
@@ -1327,27 +1283,35 @@ func (q *Queries) UpdateUser(ctx context.Context, arg UpdateUserParams) (User, e
}
const upsertDevice = `-- name: UpsertDevice :one
INSERT INTO devices (id, user_id, device_name)
VALUES (?, ?, ?)
INSERT INTO devices (id, user_id, last_synced, device_name)
VALUES (?, ?, ?, ?)
ON CONFLICT DO UPDATE
SET
device_name = COALESCE(excluded.device_name, device_name)
RETURNING id, user_id, device_name, created_at, sync
device_name = COALESCE(excluded.device_name, device_name),
last_synced = COALESCE(excluded.last_synced, last_synced)
RETURNING id, user_id, device_name, last_synced, created_at, sync
`
type UpsertDeviceParams struct {
ID string `json:"id"`
UserID string `json:"user_id"`
DeviceName string `json:"device_name"`
ID string `json:"id"`
UserID string `json:"user_id"`
LastSynced time.Time `json:"last_synced"`
DeviceName string `json:"device_name"`
}
func (q *Queries) UpsertDevice(ctx context.Context, arg UpsertDeviceParams) (Device, error) {
row := q.db.QueryRowContext(ctx, upsertDevice, arg.ID, arg.UserID, arg.DeviceName)
row := q.db.QueryRowContext(ctx, upsertDevice,
arg.ID,
arg.UserID,
arg.LastSynced,
arg.DeviceName,
)
var i Device
err := row.Scan(
&i.ID,
&i.UserID,
&i.DeviceName,
&i.LastSynced,
&i.CreatedAt,
&i.Sync,
)

327
database/schema.sql
View File

@@ -1,6 +1,10 @@
PRAGMA foreign_keys = ON;
PRAGMA journal_mode = WAL;
---------------------------------------------------------------
------------------------ Normal Tables ------------------------
---------------------------------------------------------------
-- Authentication
CREATE TABLE IF NOT EXISTS users (
id TEXT NOT NULL PRIMARY KEY,
@@ -64,27 +68,13 @@ CREATE TABLE IF NOT EXISTS devices (
user_id TEXT NOT NULL,
device_name TEXT NOT NULL,
last_synced DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
sync BOOLEAN NOT NULL DEFAULT 1 CHECK (sync IN (0, 1)),
FOREIGN KEY (user_id) REFERENCES users (id)
);
-- Document Device Sync
CREATE TABLE IF NOT EXISTS document_device_sync (
user_id TEXT NOT NULL,
document_id TEXT NOT NULL,
device_id TEXT NOT NULL,
last_synced DATETIME NOT NULL,
sync BOOLEAN NOT NULL DEFAULT 1 CHECK (sync IN (0, 1)),
FOREIGN KEY (user_id) REFERENCES users (id),
FOREIGN KEY (document_id) REFERENCES documents (id),
FOREIGN KEY (device_id) REFERENCES devices (id),
PRIMARY KEY (user_id, document_id, device_id)
);
-- User Document Progress
CREATE TABLE IF NOT EXISTS document_progress (
user_id TEXT NOT NULL,
@@ -101,17 +91,17 @@ CREATE TABLE IF NOT EXISTS document_progress (
PRIMARY KEY (user_id, document_id, device_id)
);
-- Read Activity
CREATE TABLE IF NOT EXISTS activity (
-- Raw Read Activity
CREATE TABLE IF NOT EXISTS raw_activity (
id INTEGER PRIMARY KEY AUTOINCREMENT,
user_id TEXT NOT NULL,
document_id TEXT NOT NULL,
device_id TEXT NOT NULL,
start_time DATETIME NOT NULL,
duration INTEGER NOT NULL,
page INTEGER NOT NULL,
pages INTEGER NOT NULL,
duration INTEGER NOT NULL,
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (user_id) REFERENCES users (id),
@@ -119,23 +109,71 @@ CREATE TABLE IF NOT EXISTS activity (
FOREIGN KEY (device_id) REFERENCES devices (id)
);
-- Indexes
CREATE INDEX IF NOT EXISTS activity_start_time ON activity (start_time);
CREATE INDEX IF NOT EXISTS activity_user_id_document_id ON activity (
---------------------------------------------------------------
----------------------- Temporary Tables ----------------------
---------------------------------------------------------------
-- Temporary Activity Table (Cached from View)
CREATE TEMPORARY TABLE IF NOT EXISTS activity (
user_id TEXT NOT NULL,
document_id TEXT NOT NULL,
device_id TEXT NOT NULL,
created_at DATETIME NOT NULL,
start_time DATETIME NOT NULL,
page INTEGER NOT NULL,
pages INTEGER NOT NULL,
duration INTEGER NOT NULL
);
-- Temporary User Streaks Table (Cached from View)
CREATE TEMPORARY TABLE IF NOT EXISTS user_streaks (
user_id TEXT NOT NULL,
window TEXT NOT NULL,
max_streak INTEGER NOT NULL,
max_streak_start_date TEXT NOT NULL,
max_streak_end_date TEXT NOT NULL,
current_streak INTEGER NOT NULL,
current_streak_start_date TEXT NOT NULL,
current_streak_end_date TEXT NOT NULL
);
CREATE TEMPORARY TABLE IF NOT EXISTS document_user_statistics (
document_id TEXT NOT NULL,
user_id TEXT NOT NULL,
last_read TEXT NOT NULL,
page INTEGER NOT NULL,
pages INTEGER NOT NULL,
total_time_seconds INTEGER NOT NULL,
read_pages INTEGER NOT NULL,
percentage REAL NOT NULL,
words_read INTEGER NOT NULL,
wpm REAL NOT NULL
);
---------------------------------------------------------------
--------------------------- Indexes ---------------------------
---------------------------------------------------------------
CREATE INDEX IF NOT EXISTS temp.activity_start_time ON activity (start_time);
CREATE INDEX IF NOT EXISTS temp.activity_user_id ON activity (user_id);
CREATE INDEX IF NOT EXISTS temp.activity_user_id_document_id ON activity (
user_id,
document_id
);
-- Update Trigger
CREATE TRIGGER IF NOT EXISTS update_documents_updated_at
BEFORE UPDATE ON documents BEGIN
UPDATE documents
SET updated_at = CURRENT_TIMESTAMP
WHERE id = old.id;
END;
---------------------------------------------------------------
---------------------------- Views ----------------------------
---------------------------------------------------------------
-- Rescaled Activity View (Adapted from KOReader)
CREATE VIEW IF NOT EXISTS rescaled_activity AS
--------------------------------
------- Rescaled Activity ------
--------------------------------
CREATE VIEW IF NOT EXISTS view_rescaled_activity AS
WITH RECURSIVE nums (idx) AS (
SELECT 1 AS idx
@@ -150,7 +188,7 @@ current_pages AS (
document_id,
user_id,
pages
FROM activity
FROM raw_activity
GROUP BY document_id, user_id
HAVING MAX(start_time)
ORDER BY start_time DESC
@@ -158,33 +196,33 @@ current_pages AS (
intermediate AS (
SELECT
activity.document_id,
activity.device_id,
activity.user_id,
activity.start_time,
activity.duration,
activity.page,
raw_activity.document_id,
raw_activity.device_id,
raw_activity.user_id,
raw_activity.created_at,
raw_activity.start_time,
raw_activity.duration,
raw_activity.page,
current_pages.pages,
-- Derive first page
((activity.page - 1) * current_pages.pages) / activity.pages
((raw_activity.page - 1) * current_pages.pages) / raw_activity.pages
+ 1 AS first_page,
-- Derive last page
MAX(
((activity.page - 1) * current_pages.pages)
/ activity.pages
((raw_activity.page - 1) * current_pages.pages)
/ raw_activity.pages
+ 1,
(activity.page * current_pages.pages) / activity.pages
(raw_activity.page * current_pages.pages) / raw_activity.pages
) AS last_page
FROM activity
FROM raw_activity
INNER JOIN current_pages ON
current_pages.document_id = activity.document_id
AND current_pages.user_id = activity.user_id
current_pages.document_id = raw_activity.document_id
AND current_pages.user_id = raw_activity.user_id
),
-- Improves performance
num_limit AS (
SELECT * FROM nums
LIMIT (SELECT MAX(last_page - first_page + 1) FROM intermediate)
@@ -192,28 +230,30 @@ num_limit AS (
rescaled_raw AS (
SELECT
document_id,
device_id,
user_id,
start_time,
last_page,
pages,
first_page + num_limit.idx - 1 AS page,
duration / (
last_page - first_page + 1.0
intermediate.document_id,
intermediate.device_id,
intermediate.user_id,
intermediate.created_at,
intermediate.start_time,
intermediate.last_page,
intermediate.pages,
intermediate.first_page + num_limit.idx - 1 AS page,
intermediate.duration / (
intermediate.last_page - intermediate.first_page + 1.0
) AS duration
FROM intermediate
JOIN num_limit ON
num_limit.idx <= (last_page - first_page + 1)
LEFT JOIN num_limit ON
num_limit.idx <= (intermediate.last_page - intermediate.first_page + 1)
)
SELECT
user_id,
document_id,
device_id,
user_id,
created_at,
start_time,
pages,
page,
pages,
-- Round up if last page (maintains total duration)
CAST(CASE
@@ -222,3 +262,174 @@ SELECT
ELSE duration
END AS INTEGER) AS duration
FROM rescaled_raw;
--------------------------------
--------- User Streaks ---------
--------------------------------
CREATE VIEW IF NOT EXISTS view_user_streaks AS
WITH document_windows AS (
SELECT
activity.user_id,
users.time_offset,
DATE(
activity.start_time,
users.time_offset,
'weekday 0', '-7 day'
) AS weekly_read,
DATE(activity.start_time, users.time_offset) AS daily_read
FROM raw_activity AS activity
LEFT JOIN users ON users.id = activity.user_id
GROUP BY activity.user_id, weekly_read, daily_read
),
weekly_partitions AS (
SELECT
user_id,
time_offset,
'WEEK' AS "window",
weekly_read AS read_window,
row_number() OVER (
PARTITION BY user_id ORDER BY weekly_read DESC
) AS seqnum
FROM document_windows
GROUP BY user_id, weekly_read
),
daily_partitions AS (
SELECT
user_id,
time_offset,
'DAY' AS "window",
daily_read AS read_window,
row_number() OVER (
PARTITION BY user_id ORDER BY daily_read DESC
) AS seqnum
FROM document_windows
GROUP BY user_id, daily_read
),
streaks AS (
SELECT
COUNT(*) AS streak,
MIN(read_window) AS start_date,
MAX(read_window) AS end_date,
window,
user_id,
time_offset
FROM daily_partitions
GROUP BY
time_offset,
user_id,
DATE(read_window, '+' || seqnum || ' day')
UNION ALL
SELECT
COUNT(*) AS streak,
MIN(read_window) AS start_date,
MAX(read_window) AS end_date,
window,
user_id,
time_offset
FROM weekly_partitions
GROUP BY
time_offset,
user_id,
DATE(read_window, '+' || (seqnum * 7) || ' day')
),
max_streak AS (
SELECT
MAX(streak) AS max_streak,
start_date AS max_streak_start_date,
end_date AS max_streak_end_date,
window,
user_id
FROM streaks
GROUP BY user_id, window
),
current_streak AS (
SELECT
streak AS current_streak,
start_date AS current_streak_start_date,
end_date AS current_streak_end_date,
window,
user_id
FROM streaks
WHERE CASE
WHEN window = "WEEK" THEN
DATE('now', time_offset, 'weekday 0', '-14 day') = current_streak_end_date
OR DATE('now', time_offset, 'weekday 0', '-7 day') = current_streak_end_date
WHEN window = "DAY" THEN
DATE('now', time_offset, '-1 day') = current_streak_end_date
OR DATE('now', time_offset) = current_streak_end_date
END
GROUP BY user_id, window
)
SELECT
max_streak.user_id,
max_streak.window,
IFNULL(max_streak, 0) AS max_streak,
IFNULL(max_streak_start_date, "N/A") AS max_streak_start_date,
IFNULL(max_streak_end_date, "N/A") AS max_streak_end_date,
IFNULL(current_streak, 0) AS current_streak,
IFNULL(current_streak_start_date, "N/A") AS current_streak_start_date,
IFNULL(current_streak_end_date, "N/A") AS current_streak_end_date
FROM max_streak
LEFT JOIN current_streak ON
current_streak.user_id = max_streak.user_id
AND current_streak.window = max_streak.window;
--------------------------------
------- Document Stats ---------
--------------------------------
CREATE VIEW IF NOT EXISTS view_document_user_statistics AS
WITH true_progress AS (
SELECT
document_id,
user_id,
start_time AS last_read,
page,
pages,
SUM(duration) AS total_time_seconds,
-- Determine Read Pages
COUNT(DISTINCT page) AS read_pages,
-- Derive Percentage of Book
ROUND(CAST(page AS REAL) / CAST(pages AS REAL) * 100, 2) AS percentage
FROM view_rescaled_activity
GROUP BY document_id, user_id
HAVING MAX(start_time)
)
SELECT
true_progress.*,
(CAST(COALESCE(documents.words, 0.0) AS REAL) / pages * read_pages)
AS words_read,
(CAST(COALESCE(documents.words, 0.0) AS REAL) / pages * read_pages)
/ (total_time_seconds / 60.0) AS wpm
FROM true_progress
INNER JOIN documents ON documents.id = true_progress.document_id
ORDER BY wpm DESC;
---------------------------------------------------------------
------------------ Populate Temporary Tables ------------------
---------------------------------------------------------------
INSERT INTO activity SELECT * FROM view_rescaled_activity;
INSERT INTO user_streaks SELECT * FROM view_user_streaks;
INSERT INTO document_user_statistics SELECT * FROM view_document_user_statistics;
---------------------------------------------------------------
--------------------------- Triggers --------------------------
---------------------------------------------------------------
-- Update Trigger
CREATE TRIGGER IF NOT EXISTS update_documents_updated_at
BEFORE UPDATE ON documents BEGIN
UPDATE documents
SET updated_at = CURRENT_TIMESTAMP
WHERE id = old.id;
END;

8
database/update_temp_tables.sql Normal file
View File

@@ -0,0 +1,8 @@
DELETE FROM activity;
INSERT INTO activity SELECT * FROM view_rescaled_activity;
DELETE FROM user_streaks;
INSERT INTO user_streaks SELECT * FROM view_user_streaks;
DELETE FROM document_user_statistics;
INSERT INTO document_user_statistics
SELECT *
FROM view_document_user_statistics;