diff --git a/README.md b/README.md index 7c8359f..891a476 100644 --- a/README.md +++ b/README.md @@ -42,8 +42,9 @@ In additional to the compatible KOSync API's, we add: - Additional APIs to automatically upload reading statistics - Upload documents to the server (can download in the "Documents" view or via OPDS) - Book metadata scraping (Thanks [OpenLibrary](https://openlibrary.org/) & [Google Books API](https://developers.google.com/books/docs/v1/getting_started)) -- No JavaScript for the main app! All information is generated server side with go templates. - - JavaScript is used for the ePub reader. Goals to make it service worker to enable a complete offline PWA reading experience. +- Limited JavaScript use. Server-Side Rendering is used wherever possible. The main app is fully operational without any JS. JS is only required for: + - EPUB Reader + - Offline Mode / Service Worker # Server @@ -76,25 +77,24 @@ The service is now accessible at: `http://localhost:8585`. I recommend registeri ## Configuration -| Environment Variable | Default Value | Description | -| -------------------- | ------------- | -------------------------------------------------------------------- | -| DATABASE_TYPE | SQLite | Currently only "SQLite" is supported | -| DATABASE_NAME | bbank | The database name, or in SQLite's case, the filename | -| DATABASE_PASSWORD | | Currently not used. Placeholder for potential alternative DB support | -| CONFIG_PATH | /config | Directory where to store SQLite's DB | -| DATA_PATH | /data | Directory where to store the documents and cover metadata | -| LISTEN_PORT | 8585 | Port the server listens at | -| REGISTRATION_ENABLED | false | Whether to allow registration (applies to both WebApp & KOSync API) | -| COOKIE_SESSION_KEY | | Optional secret cookie session key (auto generated if not provided) | -| COOKIE_SECURE | true | Set Cookie `Secure` attribute (i.e. only works over HTTPS) | -| COOKIE_HTTP_ONLY | true | Set Cookie `HttpOnly` attribute (i.e. inacessible via JavaScript) | +| Environment Variable | Default Value | Description | +| -------------------- | ------------- | ------------------------------------------------------------------- | +| DATABASE_TYPE | SQLite | Currently only "SQLite" is supported | +| DATABASE_NAME | book_manager | The database name, or in SQLite's case, the filename | +| CONFIG_PATH | /config | Directory where to store SQLite's DB | +| DATA_PATH | /data | Directory where to store the documents and cover metadata | +| LISTEN_PORT | 8585 | Port the server listens at | +| REGISTRATION_ENABLED | false | Whether to allow registration (applies to both WebApp & KOSync API) | +| COOKIE_SESSION_KEY | | Optional secret cookie session key (auto generated if not provided) | +| COOKIE_SECURE | true | Set Cookie `Secure` attribute (i.e. only works over HTTPS) | +| COOKIE_HTTP_ONLY | true | Set Cookie `HttpOnly` attribute (i.e. inacessible via JavaScript) | ## Security ### Authentication - _Web App / PWA_ - Session based token (7 day expiry, refresh after 6 days) -- _KOSync & SyncNinja API_ - Header based (KOSync compatibility) +- _KOSync & SyncNinja API_ - Header based - `X-Auth-User` & `X-Auth-Key` (KOSync compatibility) - _OPDS API_ - Basic authentication (KOReader OPDS compatibility) ### Notes @@ -119,7 +119,7 @@ go install github.com/sqlc-dev/sqlc/cmd/sqlc@latest Run Development: ```bash -CONFIG_PATH=./data DATA_PATH=./data go run main.go serve +CONFIG_PATH=./data DATA_PATH=./data REGISTRATION_ENABLED=true go run main.go serve ``` # Building @@ -136,6 +136,16 @@ make docker_build_local # Build Docker & Push Latest or Dev (Linux - arm64 & amd64) make docker_build_release_latest make docker_build_release_dev + +# Generate Tailwind CSS +make build_tailwind + +# Clean Local Build +make clean + +# Tests (Unit & Integration - Google Books API) +make tests_unit +make tests_integration ``` ## Notes diff --git a/api/api.go b/api/api.go index e886875..73e6471 100644 --- a/api/api.go +++ b/api/api.go @@ -80,7 +80,6 @@ func (api *API) registerWebAppRoutes() { render.AddFromFiles("error", "templates/error.html") render.AddFromFilesFuncs("login", helperFuncs, "templates/login.html") - render.AddFromFilesFuncs("reader", helperFuncs, "templates/reader-base.html", "templates/reader.html") render.AddFromFilesFuncs("home", helperFuncs, "templates/base.html", "templates/home.html") render.AddFromFilesFuncs("search", helperFuncs, "templates/base.html", "templates/search.html") render.AddFromFilesFuncs("settings", helperFuncs, "templates/base.html", "templates/settings.html") @@ -90,7 +89,15 @@ func (api *API) registerWebAppRoutes() { api.Router.HTMLRender = render + // Static Assets (Require @ Root) api.Router.GET("/manifest.json", api.webManifest) + api.Router.GET("/sw.js", api.serviceWorker) + + // Offline Static Pages (No Template) + api.Router.GET("/offline", api.offlineDocuments) + api.Router.GET("/reader", api.documentReader) + + // Template App api.Router.GET("/login", api.createAppResourcesRoute("login")) api.Router.GET("/register", api.createAppResourcesRoute("login", gin.H{"Register": true})) api.Router.GET("/logout", api.authWebAppMiddleware, api.authLogout) @@ -104,12 +111,12 @@ func (api *API) registerWebAppRoutes() { api.Router.GET("/documents", api.authWebAppMiddleware, api.createAppResourcesRoute("documents")) api.Router.POST("/documents", api.authWebAppMiddleware, api.uploadNewDocument) api.Router.GET("/documents/:document", api.authWebAppMiddleware, api.createAppResourcesRoute("document")) - api.Router.GET("/documents/:document/reader", api.authWebAppMiddleware, api.documentReader) api.Router.GET("/documents/:document/file", api.authWebAppMiddleware, api.downloadDocument) api.Router.GET("/documents/:document/cover", api.authWebAppMiddleware, api.getDocumentCover) api.Router.POST("/documents/:document/edit", api.authWebAppMiddleware, api.editDocument) api.Router.POST("/documents/:document/identify", api.authWebAppMiddleware, api.identifyDocument) api.Router.POST("/documents/:document/delete", api.authWebAppMiddleware, api.deleteDocument) + api.Router.GET("/documents/:document/progress", api.authWebAppMiddleware, api.getDocumentProgress) // Behind Configuration Flag if api.Config.SearchEnabled { diff --git a/api/app-routes.go b/api/app-routes.go index 59a5e50..8de023f 100644 --- a/api/app-routes.go +++ b/api/app-routes.go @@ -72,6 +72,18 @@ func (api *API) webManifest(c *gin.Context) { c.File("./assets/manifest.json") } +func (api *API) serviceWorker(c *gin.Context) { + c.File("./assets/sw.js") +} + +func (api *API) offlineDocuments(c *gin.Context) { + c.File("./assets/offline/index.html") +} + +func (api *API) documentReader(c *gin.Context) { + c.File("./assets/reader/index.html") +} + func (api *API) createAppResourcesRoute(routeName string, args ...map[string]any) func(*gin.Context) { // Merge Optional Template Data var templateVarsBase = gin.H{} @@ -245,7 +257,7 @@ func (api *API) getDocumentCover(c *gin.Context) { // Handle Identified Document if document.Coverfile != nil { if *document.Coverfile == "UNKNOWN" { - c.File("./assets/no-cover.jpg") + c.File("./assets/images/no-cover.jpg") return } @@ -256,7 +268,7 @@ func (api *API) getDocumentCover(c *gin.Context) { _, err = os.Stat(safePath) if err != nil { log.Error("[getDocumentCover] File Should But Doesn't Exist:", err) - c.File("./assets/no-cover.jpg") + c.File("./assets/images/no-cover.jpg") return } @@ -309,7 +321,7 @@ func (api *API) getDocumentCover(c *gin.Context) { // Return Unknown Cover if coverFile == "UNKNOWN" { - c.File("./assets/no-cover.jpg") + c.File("./assets/images/no-cover.jpg") return } @@ -317,12 +329,12 @@ func (api *API) getDocumentCover(c *gin.Context) { c.File(coverFilePath) } -func (api *API) documentReader(c *gin.Context) { +func (api *API) getDocumentProgress(c *gin.Context) { rUser, _ := c.Get("AuthorizedUser") var rDoc requestDocumentID if err := c.ShouldBindUri(&rDoc); err != nil { - log.Error("[documentReader] Invalid URI Bind") + log.Error("[getDocumentProgress] Invalid URI Bind") errorPage(c, http.StatusNotFound, "Invalid document.") return } @@ -333,7 +345,7 @@ func (api *API) documentReader(c *gin.Context) { }) if err != nil && err != sql.ErrNoRows { - log.Error("[documentReader] UpsertDocument DB Error:", err) + log.Error("[getDocumentProgress] UpsertDocument DB Error:", err) errorPage(c, http.StatusInternalServerError, fmt.Sprintf("UpsertDocument DB Error: %v", err)) return } @@ -343,15 +355,18 @@ func (api *API) documentReader(c *gin.Context) { DocumentID: rDoc.DocumentID, }) if err != nil { - log.Error("[documentReader] GetDocumentWithStats DB Error:", err) + log.Error("[getDocumentProgress] GetDocumentWithStats DB Error:", err) errorPage(c, http.StatusInternalServerError, fmt.Sprintf("GetDocumentWithStats DB Error: %v", err)) return } - c.HTML(http.StatusOK, "reader", gin.H{ - "SearchEnabled": api.Config.SearchEnabled, - "Progress": progress.Progress, - "Data": document, + c.JSON(http.StatusOK, gin.H{ + "id": document.ID, + "title": document.Title, + "author": document.Author, + "words": document.Words, + "progress": progress.Progress, + "percentage": document.Percentage, }) } diff --git a/api/ko-routes.go b/api/ko-routes.go index cccee5d..8e56815 100644 --- a/api/ko-routes.go +++ b/api/ko-routes.go @@ -636,7 +636,7 @@ func (api *API) downloadDocument(c *gin.Context) { } // Force Download (Security) - c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=%s", filepath.Base(*document.Filepath))) + c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filepath.Base(*document.Filepath))) c.File(filePath) } diff --git a/assets/book1.jpg b/assets/images/book1.jpg similarity index 100% rename from assets/book1.jpg rename to assets/images/book1.jpg diff --git a/assets/book2.jpg b/assets/images/book2.jpg similarity index 100% rename from assets/book2.jpg rename to assets/images/book2.jpg diff --git a/assets/book3.jpg b/assets/images/book3.jpg similarity index 100% rename from assets/book3.jpg rename to assets/images/book3.jpg diff --git a/assets/book4.jpg b/assets/images/book4.jpg similarity index 100% rename from assets/book4.jpg rename to assets/images/book4.jpg diff --git a/assets/no-cover.jpg b/assets/images/no-cover.jpg similarity index 100% rename from assets/no-cover.jpg rename to assets/images/no-cover.jpg diff --git a/assets/index.js b/assets/index.js new file mode 100644 index 0000000..35389d7 --- /dev/null +++ b/assets/index.js @@ -0,0 +1,78 @@ +// Install Service Worker +async function installServiceWorker() { + // Attempt Installation + await SW.install() + .then(() => console.log("[installServiceWorker] Service Worker Installed")) + .catch((e) => + console.log("[installServiceWorker] Service Worker Install Error:", e) + ); +} + +// Flush Cached Progress & Activity +async function flushCachedData() { + let allProgress = await IDB.find(/^PROGRESS-/, true); + let allActivity = await IDB.get("ACTIVITY"); + + console.log("[flushCachedData] Flushing Data:", { allProgress, allActivity }); + + Object.entries(allProgress).forEach(([id, progressEvent]) => { + flushProgress(progressEvent) + .then(() => { + console.log("[flushCachedData] Progress Flush Success:", id); + return IDB.del(id); + }) + .catch((e) => { + console.log("[flushCachedData] Progress Flush Failure:", id, e); + }); + }); + + if (!allActivity) return; + + flushActivity(allActivity) + .then(() => { + console.log("[flushCachedData] Activity Flush Success"); + return IDB.del("ACTIVITY"); + }) + .catch((e) => { + console.log("[flushCachedData] Activity Flush Failure", e); + }); +} + +function flushActivity(activityEvent) { + console.log("[flushActivity] Flushing Activity..."); + + // Flush Activity + return fetch("/api/ko/activity", { + method: "POST", + body: JSON.stringify(activityEvent), + }).then(async (r) => + console.log("[flushActivity] Flushed Activity:", { + response: r, + json: await r.json(), + data: activityEvent, + }) + ); +} + +function flushProgress(progressEvent) { + console.log("[flushProgress] Flushing Progress..."); + + // Flush Progress + return fetch("/api/ko/syncs/progress", { + method: "PUT", + body: JSON.stringify(progressEvent), + }).then(async (r) => + console.log("[flushProgress] Flushed Progress:", { + response: r, + json: await r.json(), + data: progressEvent, + }) + ); +} + +// Event Listeners +window.addEventListener("online", flushCachedData); + +// Initial Load +flushCachedData(); +installServiceWorker(); diff --git a/assets/reader/epub.min.js b/assets/lib/epub.min.js similarity index 100% rename from assets/reader/epub.min.js rename to assets/lib/epub.min.js diff --git a/assets/lib/idb-keyval.js b/assets/lib/idb-keyval.js new file mode 100644 index 0000000..9821355 --- /dev/null +++ b/assets/lib/idb-keyval.js @@ -0,0 +1,50 @@ +function _slicedToArray(t,n){return _arrayWithHoles(t)||_iterableToArrayLimit(t,n)||_unsupportedIterableToArray(t,n)||_nonIterableRest()}function _nonIterableRest(){throw new TypeError("Invalid attempt to destructure non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.")}function _unsupportedIterableToArray(t,n){if(t){if("string"==typeof t)return _arrayLikeToArray(t,n);var r=Object.prototype.toString.call(t).slice(8,-1);return"Object"===r&&t.constructor&&(r=t.constructor.name),"Map"===r||"Set"===r?Array.from(t):"Arguments"===r||/^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(r)?_arrayLikeToArray(t,n):void 0}}function _arrayLikeToArray(t,n){(null==n||n>t.length)&&(n=t.length);for(var r=0,e=new Array(n);r0&&void 0!==arguments[0]?arguments[0]:o();return t("readwrite",(function(t){return t.clear(),n(t.transaction)}))},t.createStore=r,t.del=function(t){var r=arguments.length>1&&void 0!==arguments[1]?arguments[1]:o();return r("readwrite",(function(r){return r.delete(t),n(r.transaction)}))},t.delMany=function(t){var r=arguments.length>1&&void 0!==arguments[1]?arguments[1]:o();return r("readwrite",(function(r){return t.forEach((function(t){return r.delete(t)})),n(r.transaction)}))},t.entries=function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:o();return t("readonly",(function(r){if(r.getAll&&r.getAllKeys)return Promise.all([n(r.getAllKeys()),n(r.getAll())]).then((function(t){var n=_slicedToArray(t,2),r=n[0],e=n[1];return r.map((function(t,n){return[t,e[n]]}))}));var e=[];return t("readonly",(function(t){return u(t,(function(t){return e.push([t.key,t.value])})).then((function(){return e}))}))}))},t.get=function(t){var r=arguments.length>1&&void 0!==arguments[1]?arguments[1]:o();return r("readonly",(function(r){return n(r.get(t))}))},t.getMany=function(t){var r=arguments.length>1&&void 0!==arguments[1]?arguments[1]:o();return r("readonly",(function(r){return Promise.all(t.map((function(t){return n(r.get(t))})))}))},t.keys=function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:o();return t("readonly",(function(t){if(t.getAllKeys)return n(t.getAllKeys());var r=[];return u(t,(function(t){return r.push(t.key)})).then((function(){return r}))}))},t.promisifyRequest=n,t.set=function(t,r){var e=arguments.length>2&&void 0!==arguments[2]?arguments[2]:o();return e("readwrite",(function(e){return e.put(r,t),n(e.transaction)}))},t.setMany=function(t){var r=arguments.length>1&&void 0!==arguments[1]?arguments[1]:o();return r("readwrite",(function(r){return t.forEach((function(t){return r.put(t[1],t[0])})),n(r.transaction)}))},t.update=function(t,r){var e=arguments.length>2&&void 0!==arguments[2]?arguments[2]:o();return e("readwrite",(function(e){return new Promise((function(o,u){e.get(t).onsuccess=function(){try{e.put(r(this.result),t),o(n(e.transaction))}catch(t){u(t)}}}))}))},t.values=function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:o();return t("readonly",(function(t){if(t.getAll)return n(t.getAll());var r=[];return u(t,(function(t){return r.push(t.value)})).then((function(){return r}))}))},Object.defineProperty(t,"__esModule",{value:!0})})); + +/** + * Custom IDB Convenience Functions Wrapper + **/ +const IDB = (function () { + let { get, del, entries, update, keys } = idbKeyval; + + return { + async set(key, newValue) { + let changeObj = {}; + await update(key, (oldValue) => { + if (oldValue != null) changeObj.oldValue = oldValue; + changeObj.newValue = newValue; + return newValue; + }); + return changeObj; + }, + + get(key, defaultValue) { + return get(key).then((resp) => { + return defaultValue && resp == null ? defaultValue : resp; + }); + }, + + del(key) { + return del(key); + }, + + find(keyRegExp, includeValues = false) { + if (!(keyRegExp instanceof RegExp)) throw new Error("Invalid RegExp"); + + if (!includeValues) + return keys().then((allKeys) => + allKeys.filter((key) => keyRegExp.test(key)) + ); + + return entries().then((allItems) => { + const matchingKeys = allItems.filter((keyVal) => + keyRegExp.test(keyVal[0]) + ); + return matchingKeys.reduce((obj, keyVal) => { + const [key, val] = keyVal; + obj[key] = val; + return obj; + }, {}); + }); + }, + }; +})(); diff --git a/assets/reader/jszip.min.js b/assets/lib/jszip.min.js similarity index 100% rename from assets/reader/jszip.min.js rename to assets/lib/jszip.min.js diff --git a/assets/reader/no-sleep.js b/assets/lib/no-sleep.js similarity index 100% rename from assets/reader/no-sleep.js rename to assets/lib/no-sleep.js diff --git a/assets/reader/platform.js b/assets/lib/platform.js similarity index 100% rename from assets/reader/platform.js rename to assets/lib/platform.js diff --git a/assets/lib/sw-helper.js b/assets/lib/sw-helper.js new file mode 100644 index 0000000..989cfed --- /dev/null +++ b/assets/lib/sw-helper.js @@ -0,0 +1,60 @@ +const SW = (function () { + // Helper Function + function randomID() { + return "00000000000000000000000000000000".replace(/[018]/g, (c) => + (c ^ (crypto.getRandomValues(new Uint8Array(1))[0] & (15 >> (c / 4)))) + .toString(16) + .toUpperCase() + ); + } + + // Variables + let swInstance = null; + let outstandingMessages = {}; + + navigator.serviceWorker.addEventListener("message", ({ data }) => { + let { id } = data; + data = data.data; + + console.log("[SW] Received Message:", { id, data }); + if (!outstandingMessages[id]) + return console.warn("[SW] Invalid Outstanding Message:", { id, data }); + + outstandingMessages[id](data); + delete outstandingMessages[id]; + }); + + async function install() { + // Register Service Worker + swInstance = await navigator.serviceWorker.register("/sw.js"); + swInstance.onupdatefound = (data) => + console.log("[SW.install] Update Found:", data); + + // Wait for Registration / Update + let serviceWorker = + swInstance.installing || swInstance.waiting || swInstance.active; + + // Await Installation + await new Promise((resolve) => { + serviceWorker.onstatechange = (data) => { + console.log("[SW.install] State Change:", serviceWorker.state); + if (serviceWorker.state == "activated") resolve(); + }; + if (serviceWorker.state == "activated") resolve(); + }); + } + + function send(data) { + if (!swInstance?.active) return Promise.reject("Inactive Service Worker"); + let id = randomID(); + + let msgPromise = new Promise((resolve) => { + outstandingMessages[id] = resolve; + }); + + swInstance.active.postMessage({ id, data }); + return msgPromise; + } + + return { install, send }; +})(); diff --git a/assets/offline/index.html b/assets/offline/index.html new file mode 100644 index 0000000..5bc24dc --- /dev/null +++ b/assets/offline/index.html @@ -0,0 +1,179 @@ + + + + + + + + + + + Book Manager - Offline + + + + + + + + + + + + +
+

+ Offline Documents +

+
+ +
+
+
+ You're Online: + Go Home +
+ +
+ Loading... +
+ +
+ +
+ +
+
+ + +
+ +
+ +
+
+
+ + diff --git a/assets/offline/index.js b/assets/offline/index.js new file mode 100644 index 0000000..54ff3ab --- /dev/null +++ b/assets/offline/index.js @@ -0,0 +1,142 @@ +/** + * TODO: + * - Offling / Online Checker + * - Flush oustanding read activity & progress + * - No files cached + * - Upload Files + **/ + +const BASE_ITEM = ` +
+
+
+ + + +
+
+
+
+

Title

+

+ N/A +

+
+
+
+
+

Author

+

+ N/A +

+
+
+
+
+

Progress

+

+ 0% +

+
+
+
+ +
+
`; + +const GET_SW_CACHE = "GET_SW_CACHE"; +const DEL_SW_CACHE = "DEL_SW_CACHE"; + +async function initOffline() { + updateOnlineIndicator(); + + if (document.location.pathname !== "/offline") + window.history.replaceState(null, null, "/offline"); + + // Ensure Installed + await SW.install(); + + // Get Service Worker Cache & Local Cache - Override Local + let swCache = await SW.send({ type: GET_SW_CACHE }); + let allCache = await Promise.all( + swCache.map(async (item) => { + let localCache = await IDB.get("PROGRESS-" + item.id); + if (localCache) { + item.progress = localCache.progress; + item.percentage = Math.round(localCache.percentage * 10000) / 100; + } + + return item; + }) + ); + + populateDOM(allCache); +} + +/** + * Populate DOM with cached documents. + **/ +function populateDOM(data) { + let allDocuments = document.querySelector("#items"); + + // Update Loader / No Results Indicator + let loadingEl = document.querySelector("#loading"); + if (data.length == 0) loadingEl.innerText = "No Results"; + else loadingEl.remove(); + + data.forEach((item) => { + // Create Main Element + let baseEl = document.createElement("div"); + baseEl.innerHTML = BASE_ITEM; + baseEl = baseEl.firstElementChild; + + // Get Elements + let coverEl = baseEl.querySelector("a img"); + let [titleEl, authorEl, percentageEl] = baseEl.querySelectorAll("p + p"); + let downloadEl = baseEl.querySelector("svg").parentElement; + + // Set Variables + downloadEl.setAttribute("href", "/documents/" + item.id + "/file"); + coverEl.setAttribute("src", "/documents/" + item.id + "/cover"); + coverEl.parentElement.setAttribute("href", "/reader#id=" + item.id); + titleEl.textContent = item.title; + authorEl.textContent = item.author; + percentageEl.textContent = item.percentage + "%"; + + allDocuments.append(baseEl); + }); +} + +/** + * Allow adding file to offline reader. Add to IndexedDB, + * and later upload? Add style indicating external file? + **/ +function handleFileAdd() {} + +function updateOnlineIndicator(isOnline) { + let onlineEl = document.querySelector("#online"); + isOnline = isOnline == undefined ? navigator.onLine : isOnline; + onlineEl.hidden = !isOnline; +} + +// Initialize +window.addEventListener("DOMContentLoaded", initOffline); +window.addEventListener("online", () => updateOnlineIndicator(true)); +window.addEventListener("offline", () => updateOnlineIndicator(false)); diff --git a/templates/reader-base.html b/assets/reader/index.html similarity index 92% rename from templates/reader-base.html rename to assets/reader/index.html index 5b6dec0..e57d879 100644 --- a/templates/reader-base.html +++ b/assets/reader/index.html @@ -14,11 +14,23 @@ /> - Book Manager - {{block "title" .}}{{end}} + Book Manager - Reader + + + + + + + + + + + +