[perf] dont immediately update view cache

api/app-routes.go

@@ -35,6 +35,7 @@ const (
 	adminBackup        adminAction = "BACKUP"
 	adminRestore       adminAction = "RESTORE"
 	adminMetadataMatch adminAction = "METADATA_MATCH"
+	adminCacheTables   adminAction = "CACHE_TABLES"
 )
 
 type importType string
@@ -373,6 +374,8 @@ func (api *API) appPerformAdminAction(c *gin.Context) {
 	switch rAdminAction.Action {
 	case adminImport:
 		// TODO
+	case adminCacheTables:
+		go api.DB.CacheTempTables()
 	case adminMetadataMatch:
 		// TODO
 		// 1. Documents xref most recent metadata table?

api/ko-routes.go

@@ -172,13 +172,6 @@ func (api *API) koSetProgress(c *gin.Context) {
 		return
 	}
 
-	// Update Statistic
-	start := time.Now()
-	if err := api.DB.UpdateDocumentUserStatistic(rPosition.DocumentID, auth.UserName); err != nil {
-		log.Error("[koSetProgress] UpdateDocumentUserStatistic Error:", err)
-	}
-	log.Debug("[koSetProgress] UpdateDocumentUserStatistic Performance: ", time.Since(start))
-
 	c.JSON(http.StatusOK, gin.H{
 		"document":  progress.DocumentID,
 		"timestamp": progress.CreatedAt,
@@ -301,15 +294,6 @@ func (api *API) koAddActivities(c *gin.Context) {
 		return
 	}
 
-	// Update Statistic
-	for _, doc := range allDocuments {
-		log.Info("[koAddActivities] UpdateDocumentUserStatistic Running...")
-		if err := api.DB.UpdateDocumentUserStatistic(doc, auth.UserName); err != nil {
-			log.Error("[koAddActivities] UpdateDocumentUserStatistic Error:", err)
-		}
-		log.Info("[koAddActivities] UpdateDocumentUserStatistic Complete")
-	}
-
 	c.JSON(http.StatusOK, gin.H{
 		"added": len(rActivity.Activity),
 	})

database/manager.go

@@ -5,9 +5,11 @@ import (
 	"database/sql"
 	_ "embed"
 	"fmt"
+	"path"
+	"time"
+
 	log "github.com/sirupsen/logrus"
 	_ "modernc.org/sqlite"
-	"path"
 	"reichard.io/antholume/config"
 )
 
@@ -20,12 +22,6 @@ type DBManager struct {
 //go:embed schema.sql
 var ddl string
 
-//go:embed update_temp_tables.sql
-var tsql string
-
-//go:embed update_document_user_statistics.sql
-var doc_user_stat_sql string
-
 func NewMgr(c *config.Config) *DBManager {
 	// Create Manager
 	dbm := &DBManager{
@@ -63,24 +59,26 @@ func (dbm *DBManager) Shutdown() error {
 	return dbm.DB.Close()
 }
 
-func (dbm *DBManager) UpdateDocumentUserStatistic(documentID string, userID string) error {
-	// Prepare Statement
-	stmt, err := dbm.DB.PrepareContext(dbm.Ctx, doc_user_stat_sql)
-	if err != nil {
-		return err
-	}
-	defer stmt.Close()
-
-	// Execute
-	if _, err := stmt.ExecContext(dbm.Ctx, documentID, userID); err != nil {
-		return err
-	}
-	return nil
-}
-
 func (dbm *DBManager) CacheTempTables() error {
-	if _, err := dbm.DB.ExecContext(dbm.Ctx, tsql); err != nil {
+	start := time.Now()
+	user_streaks_sql := `
+	  DELETE FROM user_streaks;
+	  INSERT INTO user_streaks SELECT * FROM view_user_streaks;
+	`
+	if _, err := dbm.DB.ExecContext(dbm.Ctx, user_streaks_sql); err != nil {
 		return err
 	}
+	log.Debug("[CacheTempTables] Cached 'user_streaks' in: ", time.Since(start))
+
+	start = time.Now()
+	document_statistics_sql := `
+	  DELETE FROM document_user_statistics;
+	  INSERT INTO document_user_statistics SELECT * FROM view_document_user_statistics;
+	`
+	if _, err := dbm.DB.ExecContext(dbm.Ctx, document_statistics_sql); err != nil {
+		return err
+	}
+	log.Debug("[CacheTempTables] Cached 'document_user_statistics' in: ", time.Since(start))
+
 	return nil
 }

database/update_document_user_statistics.sql

@@ -1,77 +0,0 @@
-INSERT INTO document_user_statistics
-WITH intermediate_ga AS (
-    SELECT
-        ga1.id AS row_id,
-        ga1.user_id,
-        ga1.document_id,
-        ga1.duration,
-        ga1.start_time,
-        ga1.start_percentage,
-        ga1.end_percentage,
-
-        -- Find Overlapping Events (Assign Unique ID)
-        (
-            SELECT MIN(id)
-            FROM activity AS ga2
-            WHERE
-                ga1.document_id = ga2.document_id
-                AND ga1.user_id = ga2.user_id
-                AND ga1.start_percentage <= ga2.end_percentage
-                AND ga1.end_percentage >= ga2.start_percentage
-        ) AS group_leader
-    FROM activity AS ga1
-    WHERE
-        document_id = ?
-        AND user_id = ?
-),
-grouped_activity AS (
-    SELECT
-        user_id,
-        document_id,
-        MAX(start_time) AS start_time,
-        MIN(start_percentage) AS start_percentage,
-        MAX(end_percentage) AS end_percentage,
-        MAX(end_percentage) - MIN(start_percentage) AS read_percentage,
-        SUM(duration) AS duration
-    FROM intermediate_ga
-    GROUP BY group_leader
-),
-current_progress AS (
-    SELECT
-        user_id,
-        document_id,
-        COALESCE((
-            SELECT percentage
-            FROM document_progress AS dp
-            WHERE
-                dp.user_id = iga.user_id
-                AND dp.document_id = iga.document_id
-	    ORDER BY created_at DESC
-	    LIMIT 1
-        ), end_percentage) AS percentage
-    FROM intermediate_ga AS iga
-    GROUP BY user_id, document_id
-    HAVING MAX(start_time)
-)
-SELECT
-    ga.document_id,
-    ga.user_id,
-    MAX(start_time) AS last_read,
-    SUM(duration) AS total_time_seconds,
-    SUM(read_percentage) AS read_percentage,
-    cp.percentage,
-
-    (CAST(COALESCE(d.words, 0.0) AS REAL) * SUM(read_percentage))
-        AS words_read,
-
-    (CAST(COALESCE(d.words, 0.0) AS REAL) * SUM(read_percentage))
-    / (SUM(duration) / 60.0) AS wpm
-FROM grouped_activity AS ga
-INNER JOIN
-    current_progress AS cp
-    ON ga.user_id = cp.user_id AND ga.document_id = cp.document_id
-INNER JOIN
-    documents AS d
-    ON d.id = ga.document_id
-GROUP BY ga.document_id, ga.user_id
-ORDER BY wpm DESC;

database/update_temp_tables.sql

@@ -1,6 +0,0 @@
-DELETE FROM user_streaks;
-INSERT INTO user_streaks SELECT * FROM view_user_streaks;
-DELETE FROM document_user_statistics;
-INSERT INTO document_user_statistics
-SELECT *
-FROM view_document_user_statistics;

server/server.go

@@ -78,11 +78,11 @@ func (s *Server) StartServer(wg *sync.WaitGroup, done <-chan struct{}) {
 }
 
 func (s *Server) RunScheduledTasks() {
-	log.Info("[RunScheduledTasks] Refreshing Temp Table Cache")
+	start := time.Now()
 	if err := s.API.DB.CacheTempTables(); err != nil {
 		log.Warn("[RunScheduledTasks] Refreshing Temp Table Cache Failure:", err)
 	}
-	log.Info("[RunScheduledTasks] Refreshing Temp Table Success")
+	log.Debug("[RunScheduledTasks] Completed in: ", time.Since(start))
 }
 
 func (s *Server) StopServer(wg *sync.WaitGroup, done chan<- struct{}) {

templates/pages/admin.html

@@ -163,15 +163,23 @@
 
           <tr>
             <td>
-              <p>Logs</p>
+              <p>Cache Tables</p>
             </td>
             <td class="py-2 float-right">
-              <a
+              <form action="./admin" method="POST">
-                href="./admin/logs"
+                <input
-                class="inline-block w-40 px-10 py-2 text-base font-semibold text-center text-white transition duration-200 ease-in bg-black shadow-md hover:text-black hover:bg-white focus:outline-none focus:ring-2"
+                  type="text"
-              >
+                  name="action"
-                <span class="w-full">View</span>
+                  value="CACHE_TABLES"
-              </a>
+                  class="hidden"
+                />
+                <button
+                  type="submit"
+                  class="w-40 px-10 py-2 text-base font-semibold text-center text-white transition duration-200 ease-in bg-black shadow-md hover:text-black hover:bg-white focus:outline-none focus:ring-2"
+                >
+                  <span class="w-full">Run</span>
+                </button>
+              </form>
             </td>
           </tr>
         </tbody>