[add] progress streaming

api/api.go

@@ -9,6 +9,7 @@ import (
 	"net/http"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/gin-contrib/multitemplate"
 	"github.com/gin-contrib/sessions"
@@ -26,15 +27,16 @@ type API struct {
 	DB         *database.DBManager
 	HTMLPolicy *bluemonday.Policy
 	Assets     *embed.FS
+	Templates  map[string]*template.Template
 }
 
-func NewApi(db *database.DBManager, c *config.Config, assets embed.FS) *API {
+func NewApi(db *database.DBManager, c *config.Config, assets *embed.FS) *API {
 	api := &API{
 		HTMLPolicy: bluemonday.StrictPolicy(),
 		Router:     gin.Default(),
 		Config:     c,
 		DB:         db,
-		Assets:     &assets,
+		Assets:     assets,
 	}
 
 	// Assets & Web App Templates
@@ -168,6 +170,7 @@ func (api *API) registerOPDSRoutes(apiGroup *gin.RouterGroup) {
 
 func (api *API) generateTemplates() *multitemplate.Renderer {
 	// Define Templates & Helper Functions
+	templates := make(map[string]*template.Template)
 	render := multitemplate.NewRenderer()
 	helperFuncs := template.FuncMap{
 		"GetSVGGraphData": getSVGGraphData,
@@ -189,6 +192,7 @@ func (api *API) generateTemplates() *multitemplate.Renderer {
 
 		b, _ := api.Assets.ReadFile(path)
 		baseTemplate = template.Must(baseTemplate.New("svg/" + name).Parse(string(b)))
+		templates["svg/"+name] = baseTemplate
 	}
 
 	// Load Components
@@ -198,8 +202,11 @@ func (api *API) generateTemplates() *multitemplate.Renderer {
 		path := fmt.Sprintf("templates/components/%s", basename)
 		name := strings.TrimSuffix(basename, filepath.Ext(basename))
 
+		// Clone Base Template
 		b, _ := api.Assets.ReadFile(path)
 		baseTemplate = template.Must(baseTemplate.New("component/" + name).Parse(string(b)))
+		render.Add("component/"+name, baseTemplate)
+		templates["component/"+name] = baseTemplate
 	}
 
 	// Load Pages
@@ -213,8 +220,11 @@ func (api *API) generateTemplates() *multitemplate.Renderer {
 		b, _ := api.Assets.ReadFile(path)
 		pageTemplate, _ := template.Must(baseTemplate.Clone()).New("page/" + name).Parse(string(b))
 		render.Add("page/"+name, pageTemplate)
+		templates["page/"+name] = pageTemplate
 	}
 
+	api.Templates = templates
+
 	return &render
 }
 

api/app-routes.go

@@ -773,22 +773,63 @@ func (api *API) saveNewDocument(c *gin.Context) {
 		return
 	}
 
+	// Render Initial Template
+	var userID string
+	if rUser, _ := c.Get("AuthorizedUser"); rUser != nil {
+		userID = rUser.(string)
+	}
+	templateVars := gin.H{
+		"RouteName":     "search",
+		"SearchEnabled": api.Config.SearchEnabled,
+		"User":          userID,
+	}
+	c.HTML(http.StatusOK, "page/search", templateVars)
+
+	// Create Streamer
+	stream := api.newStreamer(c)
+	defer stream.close()
+
+	// Stream Helper Function
+	sendDownloadMessage := func(msg string, args ...map[string]any) {
+		// Merge Defaults & Overrides
+		var templateVars = gin.H{
+			"Message":    msg,
+			"ButtonText": "Close",
+			"ButtonHref": "./search",
+		}
+		if len(args) > 0 {
+			for key := range args[0] {
+				templateVars[key] = args[0][key]
+			}
+		}
+
+		stream.send("component/download-progress", templateVars)
+	}
+
+	// Send Message
+	sendDownloadMessage("Downloading document...", gin.H{"Progress": 10})
+
 	// Save Book
 	tempFilePath, err := search.SaveBook(rDocAdd.ID, rDocAdd.Source)
 	if err != nil {
 		log.Warn("[saveNewDocument] Temp File Error: ", err)
-		errorPage(c, http.StatusInternalServerError, "Unable to save file.")
+		sendDownloadMessage("Unable to download file", gin.H{"Error": true})
 		return
 	}
 
+	// Send Message
+	sendDownloadMessage("Calculating partial MD5...", gin.H{"Progress": 60})
+
 	// Calculate Partial MD5 ID
 	partialMD5, err := utils.CalculatePartialMD5(tempFilePath)
 	if err != nil {
 		log.Warn("[saveNewDocument] Partial MD5 Error: ", err)
-		errorPage(c, http.StatusInternalServerError, "Unable to calculate partial MD5.")
-		return
+		sendDownloadMessage("Unable to calculate partial MD5", gin.H{"Error": true})
 	}
 
+	// Send Message
+	sendDownloadMessage("Saving file...", gin.H{"Progress": 60})
+
 	// Derive Extension on MIME
 	fileMime, err := mimetype.DetectFile(tempFilePath)
 	fileExtension := fileMime.Extension()
@@ -817,7 +858,7 @@ func (api *API) saveNewDocument(c *gin.Context) {
 	sourceFile, err := os.Open(tempFilePath)
 	if err != nil {
 		log.Error("[saveNewDocument] Source File Error:", err)
-		errorPage(c, http.StatusInternalServerError, "Unable to save file.")
+		sendDownloadMessage("Unable to open file", gin.H{"Error": true})
 		return
 	}
 	defer os.Remove(tempFilePath)
@@ -828,7 +869,7 @@ func (api *API) saveNewDocument(c *gin.Context) {
 	destFile, err := os.Create(safePath)
 	if err != nil {
 		log.Error("[saveNewDocument] Dest File Error:", err)
-		errorPage(c, http.StatusInternalServerError, "Unable to save file.")
+		sendDownloadMessage("Unable to create file", gin.H{"Error": true})
 		return
 	}
 	defer destFile.Close()
@@ -836,26 +877,35 @@ func (api *API) saveNewDocument(c *gin.Context) {
 	// Copy File
 	if _, err = io.Copy(destFile, sourceFile); err != nil {
 		log.Error("[saveNewDocument] Copy Temp File Error:", err)
-		errorPage(c, http.StatusInternalServerError, "Unable to save file.")
+		sendDownloadMessage("Unable to save file", gin.H{"Error": true})
 		return
 	}
 
+	// Send Message
+	sendDownloadMessage("Calculating MD5...", gin.H{"Progress": 70})
+
 	// Get MD5 Hash
 	fileHash, err := getFileMD5(safePath)
 	if err != nil {
 		log.Error("[saveNewDocument] Hash Failure:", err)
-		errorPage(c, http.StatusInternalServerError, "Unable to calculate MD5.")
+		sendDownloadMessage("Unable to calculate MD5", gin.H{"Error": true})
 		return
 	}
 
+	// Send Message
+	sendDownloadMessage("Calculating word count...", gin.H{"Progress": 80})
+
 	// Get Word Count
 	wordCount, err := metadata.GetWordCount(safePath)
 	if err != nil {
 		log.Error("[saveNewDocument] Word Count Failure:", err)
-		errorPage(c, http.StatusInternalServerError, "Unable to calculate word count.")
+		sendDownloadMessage("Unable to calculate word count", gin.H{"Error": true})
 		return
 	}
 
+	// Send Message
+	sendDownloadMessage("Saving to database...", gin.H{"Progress": 90})
+
 	// Upsert Document
 	if _, err = api.DB.Queries.UpsertDocument(api.DB.Ctx, database.UpsertDocumentParams{
 		ID:       partialMD5,
@@ -866,11 +916,16 @@ func (api *API) saveNewDocument(c *gin.Context) {
 		Words:    &wordCount,
 	}); err != nil {
 		log.Error("[saveNewDocument] UpsertDocument DB Error:", err)
-		errorPage(c, http.StatusInternalServerError, fmt.Sprintf("UpsertDocument DB Error: %v", err))
+		sendDownloadMessage("Unable to save to database", gin.H{"Error": true})
 		return
 	}
 
-	c.Redirect(http.StatusFound, fmt.Sprintf("./documents/%s", partialMD5))
+	// Send Message
+	sendDownloadMessage("Download Success", gin.H{
+		"Progress":   100,
+		"ButtonText": "Go to Book",
+		"ButtonHref": fmt.Sprintf("./documents/%s", partialMD5),
+	})
 }
 
 func (api *API) editSettings(c *gin.Context) {

api/streamer.go

@@ -0,0 +1,79 @@
+package api
+
+import (
+	"bytes"
+	"html/template"
+	"net/http"
+	"sync"
+	"time"
+
+	"github.com/gin-gonic/gin"
+)
+
+type streamer struct {
+	templates  map[string]*template.Template
+	writer     gin.ResponseWriter
+	mutex      sync.Mutex
+	completeCh chan struct{}
+}
+
+func (api *API) newStreamer(c *gin.Context) *streamer {
+	stream := &streamer{
+		templates:  api.Templates,
+		writer:     c.Writer,
+		completeCh: make(chan struct{}),
+	}
+
+	// Set Headers
+	header := stream.writer.Header()
+	header.Set("Transfer-Encoding", "chunked")
+	header.Set("Content-Type", "text/html; charset=utf-8")
+	header.Set("X-Content-Type-Options", "nosniff")
+	stream.writer.WriteHeader(http.StatusOK)
+
+	// Send Open Element Tags
+	stream.write(`
+	  <div class="absolute top-0 left-0 w-full h-full z-50">
+	    <div class="fixed top-0 left-0 bg-black opacity-50 w-screen h-screen"></div>
+	    <div id="stream-main" class="relative max-h-[95%] -translate-x-2/4 top-1/2 left-1/2 w-5/6">`)
+
+	// Keep Alive
+	go func() {
+		closeCh := stream.writer.CloseNotify()
+		for {
+			select {
+			case <-stream.completeCh:
+				return
+			case <-closeCh:
+				return
+			default:
+				stream.write("<!-- ping -->")
+				time.Sleep(2 * time.Second)
+			}
+		}
+	}()
+
+	return stream
+}
+
+func (stream *streamer) write(str string) {
+	stream.mutex.Lock()
+	stream.writer.WriteString(str)
+	stream.writer.(http.Flusher).Flush()
+	stream.mutex.Unlock()
+}
+
+func (stream *streamer) send(templateName string, templateVars gin.H) {
+	t := stream.templates[templateName]
+	buf := &bytes.Buffer{}
+	_ = t.ExecuteTemplate(buf, templateName, templateVars)
+	stream.write(buf.String())
+}
+
+func (stream *streamer) close() {
+	// Send Close Element Tags
+	stream.write(`</div></div>`)
+
+	// Close
+	close(stream.completeCh)
+}

assets/sw.js

@@ -253,6 +253,13 @@ self.addEventListener("install", function (event) {
   event.waitUntil(handleInstall(event));
 });
 
-self.addEventListener("fetch", (event) =>
-  event.respondWith(handleFetch(event))
-);
+self.addEventListener("fetch", (event) => {
+  /**
+   * Weird things happen when a service worker attempts to handle a request
+   * when the server responds with chunked transfer encoding. Right now we only
+   * use chunked encoding on POSTs. So this is to avoid processing those.
+   **/
+
+  if (event.request.method != "GET") return;
+  return event.respondWith(handleFetch(event));
+});

main.go

@@ -55,7 +55,7 @@ func cmdServer(ctx *cli.Context) error {
 	signal.Notify(interrupt, os.Interrupt, syscall.SIGTERM)
 
 	// Start Server
-	server := server.NewServer(assets)
+	server := server.NewServer(&assets)
 	server.StartServer(&wg, done)
 
 	// Wait & Close

server/server.go

@@ -23,7 +23,7 @@ type Server struct {
 	httpServer *http.Server
 }
 
-func NewServer(assets embed.FS) *Server {
+func NewServer(assets *embed.FS) *Server {
 	c := config.Load()
 	db := database.NewMgr(c)
 	api := api.NewApi(db, c, assets)

templates/base.html

@@ -19,6 +19,8 @@
     <script src="/assets/common.js"></script>
     <script src="/assets/index.js"></script>
 
+    <script src="https://cdn.tailwindcss.com"></script>
+
     <style>
       /* ----------------------------- */
       /* -------- PWA Styling -------- */

templates/components/download-progress.html

@@ -0,0 +1,40 @@
+<div
+  class="absolute -translate-y-1/2 p-4 m-auto bg-gray-700 dark:bg-gray-300 rounded-lg shadow w-full text-black dark:text-white"
+>
+  <span
+    class="inline-flex gap-2 items-center font-medium text-xs inline-block py-1 px-2 uppercase rounded-full {{ if .Error }} bg-red-500 {{ else }} bg-green-600 {{ end }}"
+  >
+    {{ if and (ne .Progress 100) (not .Error) }}
+      {{ template "svg/loading" (dict "Size" 16) }} 
+    {{ end }}
+
+    {{ .Message }}
+  </span>
+  <div class="flex flex-col gap-2 mt-2">
+    <div class="relative w-full h-4 bg-gray-300 dark:bg-gray-700 rounded-full">
+
+      {{ if .Error }}
+	<div
+	  class="absolute h-full bg-red-500 rounded-full"
+	  style="width: 100%"
+	></div>
+	<p class="absolute w-full h-full font-bold text-center text-xs">ERROR</p>
+      {{ else }}
+	<div
+	  class="absolute h-full bg-green-600 rounded-full"
+	  style="width: {{ .Progress }}%"
+	></div>
+	<p class="absolute w-full h-full font-bold text-center text-xs">
+	  {{ .Progress }}%
+	</p>
+      {{ end }}
+
+    </div>
+
+    <a
+      href="{{ .ButtonHref }}"
+      class="w-full text-center font-medium px-2 py-1 text-white bg-gray-500 dark:text-gray-800 hover:bg-gray-800 dark:hover:bg-gray-100"
+      >{{ .ButtonText }}</a
+    >
+  </div>
+</div>

templates/pages/documents.html

@@ -40,7 +40,7 @@
 <div class="grid grid-cols-1 gap-4 md:grid-cols-2 lg:grid-cols-3">
   {{range $doc := .Data }}
   <div class="w-full relative">
-    <div class="flex gap-4 w-full h-full p-4 bg-white shadow-lg dark:bg-gray-700 rounded">
+    <div class="flex gap-4 w-full h-full p-4 shadow-lg bg-white dark:bg-gray-700 rounded">
       <div class="min-w-fit my-auto h-48 relative">
 	<a href="./documents/{{$doc.ID}}">
 	  <img class="rounded object-cover h-full" src="./documents/{{$doc.ID}}/cover"></img>

templates/svgs/loading.svg

@@ -0,0 +1,45 @@
+<svg
+  width="{{ or .Size 24 }}"
+  height="{{ or .Size 24 }}"
+
+  viewBox="0 0 24 24"
+  fill="currentColor"
+  xmlns="http://www.w3.org/2000/svg"
+>
+  <style>
+    .spinner_l9ve {
+      animation: spinner_rcyq 1.2s cubic-bezier(0.52, 0.6, 0.25, 0.99) infinite;
+    }
+    .spinner_cMYp {
+      animation-delay: 0.4s;
+    }
+    .spinner_gHR3 {
+      animation-delay: 0.8s;
+    }
+    @keyframes spinner_rcyq {
+      0% {
+        transform: translate(12px, 12px) scale(0);
+        opacity: 1;
+      }
+      100% {
+        transform: translate(0, 0) scale(1);
+        opacity: 0;
+      }
+    }
+  </style>
+  <path
+    class="spinner_l9ve"
+    d="M12,1A11,11,0,1,0,23,12,11,11,0,0,0,12,1Zm0,20a9,9,0,1,1,9-9A9,9,0,0,1,12,21Z"
+    transform="translate(12, 12) scale(0)"
+  />
+  <path
+    class="spinner_l9ve spinner_cMYp"
+    d="M12,1A11,11,0,1,0,23,12,11,11,0,0,0,12,1Zm0,20a9,9,0,1,1,9-9A9,9,0,0,1,12,21Z"
+    transform="translate(12, 12) scale(0)"
+  />
+  <path
+    class="spinner_l9ve spinner_gHR3"
+    d="M12,1A11,11,0,1,0,23,12,11,11,0,0,0,12,1Zm0,20a9,9,0,1,1,9-9A9,9,0,0,1,12,21Z"
+    transform="translate(12, 12) scale(0)"
+  />
+</svg>