[fix] large files cause OOM

client/syncninja.koplugin/SyncNinjaClient.lua

@@ -1,6 +1,10 @@
 local UIManager = require("ui/uimanager")
-local socketutil = require("socketutil")
+local http = require("socket.http")
+local lfs = require("libs/libkoreader-lfs")
 local logger = require("logger")
+local ltn12 = require("ltn12")
+local socket = require("socket")
+local socketutil = require("socketutil")
 
 -- Push/Pull
 local SYNC_TIMEOUTS = {2, 5}
@@ -199,23 +203,68 @@ function SyncNinjaClient:download_document(username, password, document,
     end
 end
 
-function SyncNinjaClient:upload_document(username, password, document, file,
+function SyncNinjaClient:upload_document(username, password, document, filepath,
                                          callback)
-    self.client:reset_middlewares()
+    -- Create URL
-    self.client:enable("Format.JSON")
+    local url = self.custom_url ..
-    self.client:enable("GinClient")
+                    (self.custom_url:sub(-#"/") ~= "/" and "/" or "") ..
-    self.client:enable("SyncNinjaAuth",
+                    "api/ko/documents/" .. document .. "/file"
-                       {username = username, userkey = password})
 
-    local ok, res = pcall(function()
+    -- Track Length, Sources, and Boundary
-        return self.client:upload_document({document = document, file = file})
+    local len = 0
-    end)
+    local sources = {}
-    if ok then
+    local boundary = "-----BoundaryePkpFF7tjBAqx29L"
-        callback(res.status == 200, res.body)
+
-    else
+    -- Open File & Get Size
+    local file = io.open(filepath, "rb")
+    local file_size = lfs.attributes(filepath, 'size')
+
+    -- Insert File Start
+    local str_start = {}
+    table.insert(str_start, "--" .. boundary .. "\r\n")
+    table.insert(str_start, 'content-disposition: form-data; name="file";')
+    table.insert(str_start, 'filename="' .. "test" ..
+                     '"\r\ncontent-type: application/octet-stream\r\n\r\n')
+    str_start = table.concat(str_start)
+    table.insert(sources, ltn12.source.string(str_start))
+    len = len + #str_start
+
+    -- Insert File
+    table.insert(sources, ltn12.source.file(file))
+    len = len + file_size
+
+    -- Insert File End
+    local str_end = "\r\n"
+    table.insert(sources, ltn12.source.string(str_end))
+    len = len + #str_end
+
+    -- Insert Multipart End
+    local str = string.format("--%s--\r\n", boundary)
+    table.insert(sources, ltn12.source.string(str))
+    len = len + #str
+
+    -- Execute Request
+    logger.dbg("SyncNinja: upload_document - Uploading [" .. len .. "]:",
+               filepath)
+
+    local resp = {}
+    local code, headers, status = socket.skip(1, http.request {
+        url = url,
+        method = 'PUT',
+        headers = {
+            ["Content-Length"] = len,
+            ['Content-Type'] = "multipart/form-data; boundary=" .. boundary,
+            ['X-Auth-User'] = username,
+            ['X-Auth-Key'] = password
+        },
+        source = ltn12.source.cat(unpack(sources)),
+        sink = ltn12.sink.table(resp)
+    })
+
+    if code ~= 200 then
         logger.dbg("SyncNinjaClient:upload_document failure:", res)
-        callback(false, res.body)
     end
+    callback(code == 200, resp)
 end
 
 ------------------------------------------

client/syncninja.koplugin/main.lua

@@ -1,7 +1,6 @@
 local ConfirmBox = require("ui/widget/confirmbox")
 local DataStorage = require("datastorage")
 local Device = require("device")
-local Dispatcher = require("dispatcher")
 local DocSettings = require("docsettings")
 local InfoMessage = require("ui/widget/infomessage")
 local MultiInputDialog = require("ui/widget/multiinputdialog")
@@ -15,11 +14,6 @@ local _ = require("gettext")
 local logger = require("logger")
 local md5 = require("ffi/sha2").md5
 
--- TODO:
---   - Handle ReadHistory missing files (statistics.sqlite3, bookinfo_cache.sqlite3)
---   - Handle document uploads (Manual push only, warning saying this may take awhile)
---   - Configure activity bulk size? 1000, 5000, 10000? Separate manual settings to upload ALL?
-
 ------------------------------------------
 ------------ Helper Functions ------------
 ------------------------------------------
@@ -574,9 +568,10 @@ function SyncNinja:performSync(interactive)
     self:checkActivity(interactive)
     self:checkDocuments(interactive)
 
+    -- Notify
     if interactive == true then
         UIManager:show(InfoMessage:new{
-            text = _("SyncNinja: Manual Sync Success"),
+            text = _("SyncNinja: Manual Sync Initiated"),
             timeout = 3
         })
     end
@@ -634,7 +629,6 @@ function SyncNinja:uploadActivity(activity_data, interactive)
                     timeout = 3
                 })
             end
-
             return logger.dbg("SyncNinja: uploadActivity Error:", dump(body))
         end
     end
@@ -769,7 +763,6 @@ function SyncNinja:uploadDocumentFiles(doc_metadata, interactive)
     if self.settings.sync_document_files ~= true then return end
     if interactive ~= true then return end
 
-    -- API Callback Function
     local callback_func = function(ok, body)
         if not ok then
             UIManager:show(InfoMessage:new{
@@ -787,24 +780,30 @@ function SyncNinja:uploadDocumentFiles(doc_metadata, interactive)
             text = _("Uploading Documents - Please Wait...")
         })
 
-        -- API Client
+        UIManager:nextTick(function()
-        local SyncNinjaClient = require("SyncNinjaClient")
+            -- API Client
-        local client = SyncNinjaClient:new{
+            local SyncNinjaClient = require("SyncNinjaClient")
-            custom_url = self.settings.server,
+            local client = SyncNinjaClient:new{
-            service_spec = self.path .. "/api.json"
+                custom_url = self.settings.server,
-        }
+                service_spec = self.path .. "/api.json"
+            }
 
-        for _, v in pairs(doc_metadata) do
+            for _, v in pairs(doc_metadata) do
-            if v.filepath ~= nil then
+                if v.filepath ~= nil then
-                -- TODO: Partial File Uploads (Resolve: OOM Issue)
+                    local ok, err = pcall(client.upload_document, client,
-                local ok, err = pcall(client.upload_document, client,
+                                          self.settings.username,
-                                      self.settings.username,
+                                          self.settings.password, v.id,
-                                      self.settings.password, v.id, v.filepath,
+                                          v.filepath, callback_func)
-                                      callback_func)
+                else
+                    logger.dbg("SyncNinja: uploadDocumentFiles - no file for:",
+                               v.id)
+                end
             end
-        end
 
-        UIManager:show(InfoMessage:new{text = _("Uploading Documents Complete")})
+            UIManager:show(InfoMessage:new{
+                text = _("Uploading Documents Complete")
+            })
+        end)
     end
 
     UIManager:show(ConfirmBox:new{