From 240b3a2b676163472c44ba6e62820edc49c6971d Mon Sep 17 00:00:00 2001 From: Evan Reichard Date: Wed, 20 Sep 2023 08:26:44 -0400 Subject: [PATCH] [fix] large files cause OOM --- client/syncninja.koplugin/SyncNinjaClient.lua | 77 +++++++++++++++---- client/syncninja.koplugin/main.lua | 47 ++++++----- 2 files changed, 86 insertions(+), 38 deletions(-) diff --git a/client/syncninja.koplugin/SyncNinjaClient.lua b/client/syncninja.koplugin/SyncNinjaClient.lua index d80638b..4c5c635 100644 --- a/client/syncninja.koplugin/SyncNinjaClient.lua +++ b/client/syncninja.koplugin/SyncNinjaClient.lua @@ -1,6 +1,10 @@ local UIManager = require("ui/uimanager") -local socketutil = require("socketutil") +local http = require("socket.http") +local lfs = require("libs/libkoreader-lfs") local logger = require("logger") +local ltn12 = require("ltn12") +local socket = require("socket") +local socketutil = require("socketutil") -- Push/Pull local SYNC_TIMEOUTS = {2, 5} @@ -199,23 +203,68 @@ function SyncNinjaClient:download_document(username, password, document, end end -function SyncNinjaClient:upload_document(username, password, document, file, +function SyncNinjaClient:upload_document(username, password, document, filepath, callback) - self.client:reset_middlewares() - self.client:enable("Format.JSON") - self.client:enable("GinClient") - self.client:enable("SyncNinjaAuth", - {username = username, userkey = password}) + -- Create URL + local url = self.custom_url .. + (self.custom_url:sub(-#"/") ~= "/" and "/" or "") .. + "api/ko/documents/" .. document .. "/file" - local ok, res = pcall(function() - return self.client:upload_document({document = document, file = file}) - end) - if ok then - callback(res.status == 200, res.body) - else + -- Track Length, Sources, and Boundary + local len = 0 + local sources = {} + local boundary = "-----BoundaryePkpFF7tjBAqx29L" + + -- Open File & Get Size + local file = io.open(filepath, "rb") + local file_size = lfs.attributes(filepath, 'size') + + -- Insert File Start + local str_start = {} + table.insert(str_start, "--" .. boundary .. "\r\n") + table.insert(str_start, 'content-disposition: form-data; name="file";') + table.insert(str_start, 'filename="' .. "test" .. + '"\r\ncontent-type: application/octet-stream\r\n\r\n') + str_start = table.concat(str_start) + table.insert(sources, ltn12.source.string(str_start)) + len = len + #str_start + + -- Insert File + table.insert(sources, ltn12.source.file(file)) + len = len + file_size + + -- Insert File End + local str_end = "\r\n" + table.insert(sources, ltn12.source.string(str_end)) + len = len + #str_end + + -- Insert Multipart End + local str = string.format("--%s--\r\n", boundary) + table.insert(sources, ltn12.source.string(str)) + len = len + #str + + -- Execute Request + logger.dbg("SyncNinja: upload_document - Uploading [" .. len .. "]:", + filepath) + + local resp = {} + local code, headers, status = socket.skip(1, http.request { + url = url, + method = 'PUT', + headers = { + ["Content-Length"] = len, + ['Content-Type'] = "multipart/form-data; boundary=" .. boundary, + ['X-Auth-User'] = username, + ['X-Auth-Key'] = password + }, + source = ltn12.source.cat(unpack(sources)), + sink = ltn12.sink.table(resp) + }) + + if code ~= 200 then logger.dbg("SyncNinjaClient:upload_document failure:", res) - callback(false, res.body) end + callback(code == 200, resp) end ------------------------------------------ diff --git a/client/syncninja.koplugin/main.lua b/client/syncninja.koplugin/main.lua index a8f94d0..0eb1a60 100644 --- a/client/syncninja.koplugin/main.lua +++ b/client/syncninja.koplugin/main.lua @@ -1,7 +1,6 @@ local ConfirmBox = require("ui/widget/confirmbox") local DataStorage = require("datastorage") local Device = require("device") -local Dispatcher = require("dispatcher") local DocSettings = require("docsettings") local InfoMessage = require("ui/widget/infomessage") local MultiInputDialog = require("ui/widget/multiinputdialog") @@ -15,11 +14,6 @@ local _ = require("gettext") local logger = require("logger") local md5 = require("ffi/sha2").md5 --- TODO: --- - Handle ReadHistory missing files (statistics.sqlite3, bookinfo_cache.sqlite3) --- - Handle document uploads (Manual push only, warning saying this may take awhile) --- - Configure activity bulk size? 1000, 5000, 10000? Separate manual settings to upload ALL? - ------------------------------------------ ------------ Helper Functions ------------ ------------------------------------------ @@ -574,9 +568,10 @@ function SyncNinja:performSync(interactive) self:checkActivity(interactive) self:checkDocuments(interactive) + -- Notify if interactive == true then UIManager:show(InfoMessage:new{ - text = _("SyncNinja: Manual Sync Success"), + text = _("SyncNinja: Manual Sync Initiated"), timeout = 3 }) end @@ -634,7 +629,6 @@ function SyncNinja:uploadActivity(activity_data, interactive) timeout = 3 }) end - return logger.dbg("SyncNinja: uploadActivity Error:", dump(body)) end end @@ -769,7 +763,6 @@ function SyncNinja:uploadDocumentFiles(doc_metadata, interactive) if self.settings.sync_document_files ~= true then return end if interactive ~= true then return end - -- API Callback Function local callback_func = function(ok, body) if not ok then UIManager:show(InfoMessage:new{ @@ -787,24 +780,30 @@ function SyncNinja:uploadDocumentFiles(doc_metadata, interactive) text = _("Uploading Documents - Please Wait...") }) - -- API Client - local SyncNinjaClient = require("SyncNinjaClient") - local client = SyncNinjaClient:new{ - custom_url = self.settings.server, - service_spec = self.path .. "/api.json" - } + UIManager:nextTick(function() + -- API Client + local SyncNinjaClient = require("SyncNinjaClient") + local client = SyncNinjaClient:new{ + custom_url = self.settings.server, + service_spec = self.path .. "/api.json" + } - for _, v in pairs(doc_metadata) do - if v.filepath ~= nil then - -- TODO: Partial File Uploads (Resolve: OOM Issue) - local ok, err = pcall(client.upload_document, client, - self.settings.username, - self.settings.password, v.id, v.filepath, - callback_func) + for _, v in pairs(doc_metadata) do + if v.filepath ~= nil then + local ok, err = pcall(client.upload_document, client, + self.settings.username, + self.settings.password, v.id, + v.filepath, callback_func) + else + logger.dbg("SyncNinja: uploadDocumentFiles - no file for:", + v.id) + end end - end - UIManager:show(InfoMessage:new{text = _("Uploading Documents Complete")}) + UIManager:show(InfoMessage:new{ + text = _("Uploading Documents Complete") + }) + end) end UIManager:show(ConfirmBox:new{