[improve] web reader activity & progress tracking

api/ko-routes.go

@@ -169,6 +169,13 @@ func (api *API) setProgress(c *gin.Context) {
 		return
 	}
 
+	// Update Statistic
+	log.Info("[setProgress] UpdateDocumentUserStatistic Running...")
+	if err := api.DB.UpdateDocumentUserStatistic(rPosition.DocumentID, rUser.(string)); err != nil {
+		log.Error("[setProgress] UpdateDocumentUserStatistic Error:", err)
+	}
+	log.Info("[setProgress] UpdateDocumentUserStatistic Complete")
+
 	c.JSON(http.StatusOK, gin.H{
 		"document":  progress.DocumentID,
 		"timestamp": progress.CreatedAt,
@@ -285,6 +292,15 @@ func (api *API) addActivities(c *gin.Context) {
 		return
 	}
 
+	// Update Statistic
+	for _, doc := range allDocuments {
+		log.Info("[addActivities] UpdateDocumentUserStatistic Running...")
+		if err := api.DB.UpdateDocumentUserStatistic(doc, rUser.(string)); err != nil {
+			log.Error("[addActivities] UpdateDocumentUserStatistic Error:", err)
+		}
+		log.Info("[addActivities] UpdateDocumentUserStatistic Complete")
+	}
+
 	c.JSON(http.StatusOK, gin.H{
 		"added": len(rActivity.Activity),
 	})

assets/reader/index.js

@@ -73,7 +73,6 @@ function populateMetadata(data) {
  **/
 class EBookReader {
   bookState = {
-    currentWord: 0,
     pages: 0,
     percentage: 0,
     progress: "",
@@ -115,22 +114,18 @@ class EBookReader {
    * Load progress and generate locations
    **/
   async setupReader() {
-    // Get Word Count (If Needed)
+    // Get Word Count
-    if (this.bookState.words == 0)
+    this.bookState.words = await this.countWords();
-      this.bookState.words = await this.countWords();
 
     // Load Progress
     let { cfi } = await this.getCFIFromXPath(this.bookState.progress);
-    this.bookState.currentWord = cfi
-      ? this.bookState.percentage * (this.bookState.words / 100)
-      : 0;
 
-    let getStats = function () {
+    let getStats = async function () {
       // Start Timer
       this.bookState.pageStart = Date.now();
 
       // Get Stats
-      let stats = this.getBookStats();
+      let stats = await this.getBookStats();
       this.updateBookStatElements(stats);
     }.bind(this);
 
@@ -660,7 +655,7 @@ class EBookReader {
     this.bookState.pageStart = Date.now();
 
     // Update Stats
-    let stats = this.getBookStats();
+    let stats = await this.getBookStats();
     this.updateBookStatElements(stats);
 
     // Create Progress
@@ -674,16 +669,11 @@ class EBookReader {
     // Render Previous Page
     await this.rendition.prev();
 
-    // Update Current Word
-    let pageWords = await this.getVisibleWordCount();
-    this.bookState.currentWord -= pageWords;
-    if (this.bookState.currentWord < 0) this.bookState.currentWord = 0;
-
     // Reset Read Timer
     this.bookState.pageStart = Date.now();
 
     // Update Stats
-    let stats = this.getBookStats();
+    let stats = await this.getBookStats();
     this.updateBookStatElements(stats);
 
     // Create Progress
@@ -719,9 +709,8 @@ class EBookReader {
 
     // Update Current Word
     let pageWords = await this.getVisibleWordCount();
-    let startingWord = this.bookState.currentWord;
+    let currentWord = await this.getBookWordPosition();
     let percentRead = pageWords / this.bookState.words;
-    this.bookState.currentWord += pageWords;
 
     let pageWPM = pageWords / (elapsedTime / 60000);
     console.log("[createActivity] Page WPM:", pageWPM);
@@ -740,10 +729,10 @@ class EBookReader {
 
     // Exclude 0 Pages
     if (totalPages == 0)
-      return console.log("[createActivity] Invalid Total Pages (0)");
+      return console.warn("[createActivity] Invalid Total Pages (0)");
 
     let currentPage = Math.round(
-      (startingWord * totalPages) / this.bookState.words
+      (currentWord * totalPages) / this.bookState.words
     );
 
     // Create Activity Event
@@ -805,6 +794,8 @@ class EBookReader {
     // Update Pointers
     let currentCFI = await this.rendition.currentLocation();
     let { element, xpath } = await this.getXPathFromCFI(currentCFI.start.cfi);
+    let currentWord = await this.getBookWordPosition();
+    console.log("[createProgress] Current Word:", currentWord);
     this.bookState.progress = xpath;
     this.bookState.progressElement = element;
 
@@ -814,9 +805,7 @@ class EBookReader {
       device_id: this.readerSettings.deviceID,
       device: this.readerSettings.deviceName,
       percentage:
-        Math.round(
+        Math.round((currentWord / this.bookState.words) * 100000) / 100000,
-          (this.bookState.currentWord / this.bookState.words) * 100000
-        ) / 100000,
       progress: this.bookState.progress,
     };
 
@@ -885,12 +874,13 @@ class EBookReader {
   /**
    * Get chapter pages, name and progress percentage
    **/
-  getBookStats() {
+  async getBookStats() {
     let currentProgress = this.sectionProgress();
     if (!currentProgress) return;
     let { sectionPages, sectionCurrentPage } = currentProgress;
 
     let currentLocation = this.rendition.currentLocation();
+    let currentWord = await this.getBookWordPosition();
 
     let currentTOC = this.book.navigation.toc.find(
       (item) => item.href == currentLocation.start.href
@@ -901,9 +891,7 @@ class EBookReader {
       sectionTotalPages: sectionPages,
       chapterName: currentTOC ? currentTOC.label.trim() : "N/A",
       percentage:
-        Math.round(
+        Math.round((currentWord / this.bookState.words) * 10000) / 100,
-          (this.bookState.currentWord / this.bookState.words) * 10000
-        ) / 100,
     };
   }
 
@@ -1076,6 +1064,43 @@ class EBookReader {
    * Get visible word count - used for reading stats
    **/
   async getVisibleWordCount() {
+    let visibleText = await this.getVisibleText();
+    return visibleText.trim().split(/\s+/).length;
+  }
+
+  /**
+   * Gets the word number of the whole book for the first visible word.
+   **/
+  async getBookWordPosition() {
+    // Get Contents & Spine
+    let contents = this.rendition.getContents()[0];
+    let spineItem = this.book.spine.get(contents.sectionIndex);
+
+    // Get CFI Range
+    let firstCFI = spineItem.cfiFromElement(
+      spineItem.document.body.children[0]
+    );
+    let currentLocation = await this.rendition.currentLocation();
+    let cfiRange = this.getCFIRange(firstCFI, currentLocation.start.cfi);
+
+    // Get Chapter Text (Before Current Position)
+    let textRange = await this.book.getRange(cfiRange);
+    let chapterText = textRange.toString();
+
+    // Get Chapter & Book Positions
+    let chapterWordPosition = chapterText.trim().split(/\s+/).length;
+    let preChapterWordPosition = this.book.spine.spineItems
+      .slice(0, contents.sectionIndex)
+      .reduce((totalCount, item) => totalCount + item.wordCount, 0);
+
+    // Return Current Word Pointer
+    return chapterWordPosition + preChapterWordPosition;
+  }
+
+  /**
+   * Get visible text - used for word counts
+   **/
+  async getVisibleText() {
     // Force Expand & Resize (Race Condition Issue)
     this.rendition.manager.visible().forEach((item) => item.expand());
 
@@ -1092,7 +1117,7 @@ class EBookReader {
     let visibleText = textRange.toString();
 
     // Split on Whitespace
-    return visibleText.trim().split(/\s+/).length;
+    return visibleText;
   }
 
   /**
@@ -1147,14 +1172,17 @@ class EBookReader {
    * of progress percentage. Implementation returns the same number as the
    * server side implementation.
    **/
-  countWords() {
+  async countWords() {
-    // Iterate over each item in the spine, render, and count words.
+    let spineWC = await Promise.all(
-    return this.book.spine.spineItems.reduce(async (totalCount, item) => {
+      this.book.spine.spineItems.map(async (item) => {
-      let currentCount = await totalCount;
+        let newDoc = await item.load(this.book.load.bind(this.book));
-      let newDoc = await item.load(this.book.load.bind(this.book));
+        let spineWords = newDoc.innerText.trim().split(/\s+/).length;
-      let itemCount = newDoc.innerText.trim().split(/\s+/).length;
+        item.wordCount = spineWords;
-      return currentCount + itemCount;
+        return spineWords;
-    }, 0);
+      })
+    );
+
+    return spineWC.reduce((totalCount, itemCount) => totalCount + itemCount, 0);
   }
 
   /**

database/manager.go

@@ -23,6 +23,9 @@ var ddl string
 //go:embed update_temp_tables.sql
 var tsql string
 
+//go:embed update_document_user_statistics.sql
+var doc_user_stat_sql string
+
 func NewMgr(c *config.Config) *DBManager {
 	// Create Manager
 	dbm := &DBManager{
@@ -60,6 +63,21 @@ func (dbm *DBManager) Shutdown() error {
 	return dbm.DB.Close()
 }
 
+func (dbm *DBManager) UpdateDocumentUserStatistic(documentID string, userID string) error {
+	// Prepare Statement
+	stmt, err := dbm.DB.PrepareContext(dbm.Ctx, doc_user_stat_sql)
+	if err != nil {
+		return err
+	}
+	defer stmt.Close()
+
+	// Execute
+	if _, err := stmt.ExecContext(dbm.Ctx, documentID, userID); err != nil {
+		return err
+	}
+	return nil
+}
+
 func (dbm *DBManager) CacheTempTables() error {
 	if _, err := dbm.DB.ExecContext(dbm.Ctx, tsql); err != nil {
 		return err

database/schema.sql

@@ -136,7 +136,9 @@ CREATE TEMPORARY TABLE IF NOT EXISTS document_user_statistics (
     read_percentage REAL NOT NULL,
     percentage REAL NOT NULL,
     words_read INTEGER NOT NULL,
-    wpm REAL NOT NULL
+    wpm REAL NOT NULL,
+
+    UNIQUE(document_id, user_id) ON CONFLICT REPLACE
 );
 
 
@@ -325,6 +327,8 @@ current_progress AS (
             WHERE
                 dp.user_id = iga.user_id
                 AND dp.document_id = iga.document_id
+	    ORDER BY created_at DESC
+	    LIMIT 1
         ), end_percentage) AS percentage
     FROM intermediate_ga AS iga
     GROUP BY user_id, document_id

database/update_document_user_statistics.sql

@@ -0,0 +1,77 @@
+INSERT INTO document_user_statistics
+WITH intermediate_ga AS (
+    SELECT
+        ga1.id AS row_id,
+        ga1.user_id,
+        ga1.document_id,
+        ga1.duration,
+        ga1.start_time,
+        ga1.start_percentage,
+        ga1.end_percentage,
+
+        -- Find Overlapping Events (Assign Unique ID)
+        (
+            SELECT MIN(id)
+            FROM activity AS ga2
+            WHERE
+                ga1.document_id = ga2.document_id
+                AND ga1.user_id = ga2.user_id
+                AND ga1.start_percentage <= ga2.end_percentage
+                AND ga1.end_percentage >= ga2.start_percentage
+        ) AS group_leader
+    FROM activity AS ga1
+    WHERE
+        document_id = ?
+        AND user_id = ?
+),
+grouped_activity AS (
+    SELECT
+        user_id,
+        document_id,
+        MAX(start_time) AS start_time,
+        MIN(start_percentage) AS start_percentage,
+        MAX(end_percentage) AS end_percentage,
+        MAX(end_percentage) - MIN(start_percentage) AS read_percentage,
+        SUM(duration) AS duration
+    FROM intermediate_ga
+    GROUP BY group_leader
+),
+current_progress AS (
+    SELECT
+        user_id,
+        document_id,
+        COALESCE((
+            SELECT percentage
+            FROM document_progress AS dp
+            WHERE
+                dp.user_id = iga.user_id
+                AND dp.document_id = iga.document_id
+	    ORDER BY created_at DESC
+	    LIMIT 1
+        ), end_percentage) AS percentage
+    FROM intermediate_ga AS iga
+    GROUP BY user_id, document_id
+    HAVING MAX(start_time)
+)
+SELECT
+    ga.document_id,
+    ga.user_id,
+    MAX(start_time) AS last_read,
+    SUM(duration) AS total_time_seconds,
+    SUM(read_percentage) AS read_percentage,
+    cp.percentage,
+
+    (CAST(COALESCE(d.words, 0.0) AS REAL) * SUM(read_percentage))
+        AS words_read,
+
+    (CAST(COALESCE(d.words, 0.0) AS REAL) * SUM(read_percentage))
+    / (SUM(duration) / 60.0) AS wpm
+FROM grouped_activity AS ga
+INNER JOIN
+    current_progress AS cp
+    ON ga.user_id = cp.user_id AND ga.document_id = cp.document_id
+INNER JOIN
+    documents AS d
+    ON d.id = ga.document_id
+GROUP BY ga.document_id, ga.user_id
+ORDER BY wpm DESC;