feat(db): button up migrations
All checks were successful
continuous-integration/drone/push Build is passing

This commit is contained in:
Evan Reichard 2024-02-01 20:05:35 -05:00
parent 4a5464853b
commit 5865fe3c13
11 changed files with 197 additions and 134 deletions

View File

@ -77,6 +77,7 @@ func (api *API) appPerformAdminAction(c *gin.Context) {
go api.db.CacheTempTables()
case adminRestore:
api.processRestoreFile(rAdminAction, c)
return
case adminBackup:
// Vacuum
_, err := api.db.DB.ExecContext(api.db.Ctx, "VACUUM;")
@ -202,7 +203,7 @@ func (api *API) appGetAdminLogs(c *gin.Context) {
}
templateVars["Data"] = logLines
templateVars["Filter"] = strings.TrimSpace(rAdminLogs.Filter)
templateVars["Filter"] = rAdminLogs.Filter
c.HTML(http.StatusOK, "page/admin-logs", templateVars)
}
@ -414,18 +415,22 @@ func (api *API) processRestoreFile(rAdminAction requestAdminAction, c *gin.Conte
if err != nil {
appErrorPage(c, http.StatusInternalServerError, "Unable to restore data.")
log.Panic("Unable to restore data: ", err)
return
}
// Reinit DB
if err := api.db.Reload(); err != nil {
appErrorPage(c, http.StatusInternalServerError, "Unable to reload DB.")
log.Panicf("Unable to reload DB: %v", err)
}
// Rotate Auth Hashes
if err := api.rotateAllAuthHashes(); err != nil {
appErrorPage(c, http.StatusInternalServerError, "Unable to rotate hashes.")
log.Panicf("Unable to rotate auth hashes: %v", err)
}
// Redirect to login page
c.Redirect(http.StatusFound, "/login")
}
func (api *API) restoreData(zipReader *zip.Reader) error {
@ -453,8 +458,6 @@ func (api *API) restoreData(zipReader *zip.Reader) error {
fmt.Println("Error copying file contents:", err)
return err
}
fmt.Printf("Extracted: %s\n", destPath)
}
return nil

View File

@ -141,7 +141,7 @@ class EBookReader {
return "00000000000000000000000000000000".replace(/[018]/g, (c) =>
(c ^ (crypto.getRandomValues(new Uint8Array(1))[0] & (15 >> (c / 4))))
.toString(16)
.toUpperCase()
.toUpperCase(),
);
}
@ -244,7 +244,7 @@ class EBookReader {
initThemes() {
// Register Themes
THEMES.forEach((theme) =>
this.rendition.themes.register(theme, THEME_FILE)
this.rendition.themes.register(theme, THEME_FILE),
);
let themeLinkEl = document.createElement("link");
@ -270,12 +270,12 @@ class EBookReader {
// Set Fonts
this.rendition.getContents().forEach((c) => {
let el = c.document.head.appendChild(
c.document.createElement("link")
c.document.createElement("link"),
);
el.setAttribute("rel", "stylesheet");
el.setAttribute("href", "/assets/reader/fonts.css");
});
}.bind(this)
}.bind(this),
);
}
@ -304,7 +304,7 @@ class EBookReader {
let themeColorEl = document.querySelector("[name='theme-color']");
let themeStyleSheet = document.querySelector("#themes").sheet;
let themeStyleRule = Array.from(themeStyleSheet.cssRules).find(
(item) => item.selectorText == "." + colorScheme
(item) => item.selectorText == "." + colorScheme,
);
// Match Reader Theme
@ -318,13 +318,13 @@ class EBookReader {
// Set Font Family
item.document.documentElement.style.setProperty(
"--editor-font-family",
fontFamily
fontFamily,
);
// Set Font Size
item.document.documentElement.style.setProperty(
"--editor-font-size",
fontSize + "em"
fontSize + "em",
);
// Set Highlight Style
@ -357,7 +357,7 @@ class EBookReader {
// Compute Style
let backgroundColor = getComputedStyle(
this.bookState.progressElement.ownerDocument.body
this.bookState.progressElement.ownerDocument.body,
).backgroundColor;
// Set Style
@ -438,7 +438,7 @@ class EBookReader {
touchStartX = event.changedTouches[0].screenX;
touchStartY = event.changedTouches[0].screenY;
},
false
false,
);
renderDoc.addEventListener(
@ -448,7 +448,7 @@ class EBookReader {
touchEndY = event.changedTouches[0].screenY;
handleGesture(event);
},
false
false,
);
function handleGesture(event) {
@ -512,7 +512,7 @@ class EBookReader {
bottomBar.classList.remove("bottom-0");
topBar.classList.remove("top-0");
}
}.bind(this)
}.bind(this),
);
renderDoc.addEventListener(
@ -526,7 +526,7 @@ class EBookReader {
handleSwipeDown();
return true;
}
}, 400)
}, 400),
);
function handleSwipeDown() {
@ -560,7 +560,7 @@ class EBookReader {
// "t" Key (Theme Cycle)
if ((e.keyCode || e.which) == 84) {
let currentThemeIdx = THEMES.indexOf(
readerSettings.theme.colorScheme
readerSettings.theme.colorScheme,
);
let colorScheme =
THEMES.length == currentThemeIdx + 1
@ -569,7 +569,7 @@ class EBookReader {
setTheme({ colorScheme });
}
},
false
false,
);
});
}
@ -601,7 +601,7 @@ class EBookReader {
// "t" Key (Theme Cycle)
if ((e.keyCode || e.which) == 84) {
let currentThemeIdx = THEMES.indexOf(
this.readerSettings.theme.colorScheme
this.readerSettings.theme.colorScheme,
);
let colorScheme =
THEMES.length == currentThemeIdx + 1
@ -610,7 +610,7 @@ class EBookReader {
this.setTheme({ colorScheme });
}
}.bind(this),
false
false,
);
// Color Scheme Switcher
@ -621,9 +621,9 @@ class EBookReader {
function (event) {
let colorScheme = event.target.innerText;
this.setTheme({ colorScheme });
}.bind(this)
}.bind(this),
);
}.bind(this)
}.bind(this),
);
// Font Switcher
@ -638,9 +638,9 @@ class EBookReader {
this.setTheme({ fontFamily });
this.setPosition(cfi);
}.bind(this)
}.bind(this),
);
}.bind(this)
}.bind(this),
);
// Font Size
@ -663,9 +663,9 @@ class EBookReader {
// Restore CFI
this.setPosition(cfi);
}.bind(this)
}.bind(this),
);
}.bind(this)
}.bind(this),
);
// Close Top Bar
@ -752,7 +752,7 @@ class EBookReader {
if (pageWPM >= WPM_MAX)
return console.log(
"[createActivity] Page WPM Exceeds Max (2000):",
pageWPM
pageWPM,
);
// Ensure WPM Minimum
@ -765,7 +765,7 @@ class EBookReader {
return console.warn("[createActivity] Invalid Total Pages (0)");
let currentPage = Math.round(
(currentWord * totalPages) / this.bookState.words
(currentWord * totalPages) / this.bookState.words,
);
// Create Activity Event
@ -819,7 +819,7 @@ class EBookReader {
response: r,
json: await r.json(),
data: activityEvent,
})
}),
);
}
@ -880,7 +880,7 @@ class EBookReader {
response: r,
json: await r.json(),
data: progressEvent,
})
}),
);
}
@ -916,7 +916,7 @@ class EBookReader {
let currentWord = await this.getBookWordPosition();
let currentTOC = this.book.navigation.toc.find(
(item) => item.href == currentLocation.start.href
(item) => item.href == currentLocation.start.href,
);
return {
@ -953,7 +953,7 @@ class EBookReader {
let startCFI = cfi.replace("epubcfi(", "");
let docFragmentIndex =
this.book.spine.spineItems.find((item) =>
startCFI.startsWith(item.cfiBase)
startCFI.startsWith(item.cfiBase),
).index + 1;
// Base Progress
@ -1101,7 +1101,7 @@ class EBookReader {
} else {
return null;
}
}
},
);
/**
@ -1146,7 +1146,7 @@ class EBookReader {
// Get CFI Range
let firstCFI = spineItem.cfiFromElement(
spineItem.document.body.children[0]
spineItem.document.body.children[0],
);
let currentLocation = await this.rendition.currentLocation();
let cfiRange = this.getCFIRange(firstCFI, currentLocation.start.cfi);
@ -1247,7 +1247,7 @@ class EBookReader {
let spineWords = newDoc.innerText.trim().split(/\s+/).length;
item.wordCount = spineWords;
return spineWords;
})
}),
);
return spineWC.reduce((totalCount, itemCount) => totalCount + itemCount, 0);
@ -1266,7 +1266,7 @@ class EBookReader {
**/
loadSettings() {
this.readerSettings = JSON.parse(
localStorage.getItem("readerSettings") || "{}"
localStorage.getItem("readerSettings") || "{}",
);
}
}

View File

@ -46,53 +46,60 @@ func NewMgr(c *config.Config) *DBManager {
// Init manager
func (dbm *DBManager) init() error {
if dbm.cfg.DBType == "sqlite" || dbm.cfg.DBType == "memory" {
var dbLocation string = ":memory:"
if dbm.cfg.DBType == "sqlite" {
dbLocation = filepath.Join(dbm.cfg.ConfigPath, fmt.Sprintf("%s.db", dbm.cfg.DBName))
}
var err error
dbm.DB, err = sql.Open("sqlite", dbLocation)
if err != nil {
log.Errorf("Unable to open DB: %v", err)
return err
}
// Single Open Connection
dbm.DB.SetMaxOpenConns(1)
// Execute DDL
if _, err := dbm.DB.Exec(ddl, nil); err != nil {
log.Errorf("Error executing schema: %v", err)
return err
}
// Perform Migrations
err = dbm.performMigrations()
if err != nil && err != goose.ErrNoMigrationFiles {
log.Errorf("Error running DB migrations: %v", err)
return err
}
// Set SQLite Settings (After Migrations)
pragmaQuery := `
PRAGMA foreign_keys = ON;
PRAGMA journal_mode = WAL;
`
if _, err := dbm.DB.Exec(pragmaQuery, nil); err != nil {
log.Errorf("Error executing pragma: %v", err)
return err
}
// Cache Tables
dbm.CacheTempTables()
} else {
// Build DB Location
var dbLocation string
switch dbm.cfg.DBType {
case "sqlite":
dbLocation = filepath.Join(dbm.cfg.ConfigPath, fmt.Sprintf("%s.db", dbm.cfg.DBName))
case "memory":
dbLocation = ":memory:"
default:
return fmt.Errorf("unsupported database")
}
var err error
dbm.DB, err = sql.Open("sqlite", dbLocation)
if err != nil {
log.Panicf("Unable to open DB: %v", err)
return err
}
// Single open connection
dbm.DB.SetMaxOpenConns(1)
// Check if DB is new
isNew, err := isEmpty(dbm.DB)
if err != nil {
log.Panicf("Unable to determine db info: %v", err)
return err
}
// Init SQLc
dbm.Queries = New(dbm.DB)
// Execute schema
if _, err := dbm.DB.Exec(ddl, nil); err != nil {
log.Panicf("Error executing schema: %v", err)
return err
}
// Perform migrations
err = dbm.performMigrations(isNew)
if err != nil && err != goose.ErrNoMigrationFiles {
log.Panicf("Error running DB migrations: %v", err)
return err
}
// Update settings
err = dbm.updateSettings()
if err != nil {
log.Panicf("Error running DB settings update: %v", err)
return err
}
// Cache tables
go dbm.CacheTempTables()
return nil
}
@ -136,12 +143,50 @@ func (dbm *DBManager) CacheTempTables() error {
return nil
}
func (dbm *DBManager) performMigrations() error {
// Set DB Migration
func (dbm *DBManager) updateSettings() error {
// Set SQLite PRAGMA Settings
pragmaQuery := `
PRAGMA foreign_keys = ON;
PRAGMA journal_mode = WAL;
`
if _, err := dbm.DB.Exec(pragmaQuery, nil); err != nil {
log.Errorf("Error executing pragma: %v", err)
return err
}
// Update Antholume Version in DB
if _, err := dbm.Queries.UpdateSettings(dbm.Ctx, UpdateSettingsParams{
Name: "version",
Value: dbm.cfg.Version,
}); err != nil {
log.Errorf("Error updating DB settings: %v", err)
return err
}
return nil
}
func (dbm *DBManager) performMigrations(isNew bool) error {
// Create context
ctx := context.WithValue(context.Background(), "isNew", isNew)
// Set DB migration
goose.SetBaseFS(migrations)
// Run Migrations
// Run migrations
goose.SetLogger(log.StandardLogger())
goose.SetDialect("sqlite")
return goose.Up(dbm.DB, "migrations")
if err := goose.SetDialect("sqlite"); err != nil {
return err
}
return goose.UpContext(ctx, dbm.DB, "migrations")
}
func isEmpty(db *sql.DB) (bool, error) {
var tableCount int
err := db.QueryRow("SELECT COUNT(*) FROM sqlite_master WHERE type='table';").Scan(&tableCount)
if err != nil {
return false, err
}
return tableCount == 0, nil
}

View File

@ -14,16 +14,14 @@ func init() {
}
func upUserAuthHash(ctx context.Context, tx *sql.Tx) error {
// Validate column doesn't already exist
hasCol, err := hasColumn(tx, "users", "auth_hash")
if err != nil {
return err
} else if hasCol {
// Determine if we have a new DB or not
isNew := ctx.Value("isNew").(bool)
if isNew {
return nil
}
// Copy table & create column
_, err = tx.Exec(`
_, err := tx.Exec(`
-- Create Copy Table
CREATE TABLE temp_users AS SELECT * FROM users;
ALTER TABLE temp_users ADD COLUMN auth_hash TEXT;

View File

@ -1,9 +1,9 @@
# DB Migrations
```bash
# SQL migration
goose create migration_name sql
# Go migration
goose create migration_name
```
## Note
Since we update both the `schema.sql`, as well as the migration files, when we create a new DB it will inherently be up-to-date. We don't want to run the migrations if it's already up-to-date. Instead each migration checks if we have a new DB (via a value passed into the context), and if we do we simply return.

View File

@ -1,38 +0,0 @@
package migrations
import (
"database/sql"
"fmt"
)
type columnInfo struct {
CID int
Name string
Type string
NotNull int
DefaultVal sql.NullString
PK int
}
func hasColumn(tx *sql.Tx, table string, column string) (bool, error) {
rows, err := tx.Query(fmt.Sprintf("PRAGMA table_info(%s)", table))
if err != nil {
return false, err
}
defer rows.Close()
colExists := false
for rows.Next() {
var col columnInfo
if err := rows.Scan(&col.CID, &col.Name, &col.Type, &col.NotNull, &col.DefaultVal, &col.PK); err != nil {
return false, err
}
if col.Name == column {
colExists = true
break
}
}
return colExists, nil
}

View File

@ -93,6 +93,13 @@ type Metadatum struct {
CreatedAt string `json:"created_at"`
}
type Setting struct {
ID int64 `json:"id"`
Name string `json:"name"`
Value string `json:"value"`
CreatedAt string `json:"created_at"`
}
type User struct {
ID string `json:"id"`
Pass *string `json:"-"`

View File

@ -373,6 +373,15 @@ SET
WHERE id = $user_id
RETURNING *;
-- name: UpdateSettings :one
INSERT INTO settings (name, value)
VALUES (?, ?)
ON CONFLICT DO UPDATE
SET
name = COALESCE(excluded.name, name),
value = COALESCE(excluded.value, value)
RETURNING *;
-- name: UpsertDevice :one
INSERT INTO devices (id, user_id, last_synced, device_name)
VALUES (?, ?, ?, ?)

View File

@ -1213,6 +1213,33 @@ func (q *Queries) UpdateProgress(ctx context.Context, arg UpdateProgressParams)
return i, err
}
const updateSettings = `-- name: UpdateSettings :one
INSERT INTO settings (name, value)
VALUES (?, ?)
ON CONFLICT DO UPDATE
SET
name = COALESCE(excluded.name, name),
value = COALESCE(excluded.value, value)
RETURNING id, name, value, created_at
`
type UpdateSettingsParams struct {
Name string `json:"name"`
Value string `json:"value"`
}
func (q *Queries) UpdateSettings(ctx context.Context, arg UpdateSettingsParams) (Setting, error) {
row := q.db.QueryRowContext(ctx, updateSettings, arg.Name, arg.Value)
var i Setting
err := row.Scan(
&i.ID,
&i.Name,
&i.Value,
&i.CreatedAt,
)
return i, err
}
const updateUser = `-- name: UpdateUser :one
UPDATE users
SET

View File

@ -44,7 +44,6 @@ CREATE TABLE IF NOT EXISTS documents (
-- Metadata
CREATE TABLE IF NOT EXISTS metadata (
id INTEGER PRIMARY KEY AUTOINCREMENT,
document_id TEXT NOT NULL,
title TEXT,
@ -108,6 +107,16 @@ CREATE TABLE IF NOT EXISTS activity (
FOREIGN KEY (device_id) REFERENCES devices (id)
);
-- Settings
CREATE TABLE IF NOT EXISTS settings (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
value TEXT NOT NULL,
created_at DATETIME NOT NULL DEFAULT (STRFTIME('%Y-%m-%dT%H:%M:%SZ', 'now'))
);
---------------------------------------------------------------
----------------------- Temporary Tables ----------------------
---------------------------------------------------------------

View File

@ -8,7 +8,10 @@
<div class="flex flex-col grow gap-2 p-4 rounded shadow-lg bg-white dark:bg-gray-700 text-gray-500 dark:text-white">
<p class="text-lg font-semibold text-gray-500">Selected Import Directory</p>
<form class="flex gap-4 flex-col" action="./import" method="POST">
<input type="text" name="directory" value="{{ .SelectedDirectory }}" class="hidden" />
<input type="text"
name="directory"
value="{{ .SelectedDirectory }}"
class="hidden" />
<div class="flex justify-between gap-4 w-full">
<div class="flex gap-4 items-center">
<span>{{ template "svg/import" }}</span>
@ -53,7 +56,7 @@
{{ end }}
{{ if not .Data }}
<tr>
<td class="text-center p-3" colspan="2">No Folder</td>
<td class="text-center p-3" colspan="2">No Folders</td>
</tr>
{{ end }}
{{ range $item := .Data }}