[fix] large files cause OOM
This commit is contained in:
parent
d02f8c324f
commit
240b3a2b67
@ -1,6 +1,10 @@
|
||||
local UIManager = require("ui/uimanager")
|
||||
local socketutil = require("socketutil")
|
||||
local http = require("socket.http")
|
||||
local lfs = require("libs/libkoreader-lfs")
|
||||
local logger = require("logger")
|
||||
local ltn12 = require("ltn12")
|
||||
local socket = require("socket")
|
||||
local socketutil = require("socketutil")
|
||||
|
||||
-- Push/Pull
|
||||
local SYNC_TIMEOUTS = {2, 5}
|
||||
@ -199,23 +203,68 @@ function SyncNinjaClient:download_document(username, password, document,
|
||||
end
|
||||
end
|
||||
|
||||
function SyncNinjaClient:upload_document(username, password, document, file,
|
||||
function SyncNinjaClient:upload_document(username, password, document, filepath,
|
||||
callback)
|
||||
self.client:reset_middlewares()
|
||||
self.client:enable("Format.JSON")
|
||||
self.client:enable("GinClient")
|
||||
self.client:enable("SyncNinjaAuth",
|
||||
{username = username, userkey = password})
|
||||
-- Create URL
|
||||
local url = self.custom_url ..
|
||||
(self.custom_url:sub(-#"/") ~= "/" and "/" or "") ..
|
||||
"api/ko/documents/" .. document .. "/file"
|
||||
|
||||
local ok, res = pcall(function()
|
||||
return self.client:upload_document({document = document, file = file})
|
||||
end)
|
||||
if ok then
|
||||
callback(res.status == 200, res.body)
|
||||
else
|
||||
-- Track Length, Sources, and Boundary
|
||||
local len = 0
|
||||
local sources = {}
|
||||
local boundary = "-----BoundaryePkpFF7tjBAqx29L"
|
||||
|
||||
-- Open File & Get Size
|
||||
local file = io.open(filepath, "rb")
|
||||
local file_size = lfs.attributes(filepath, 'size')
|
||||
|
||||
-- Insert File Start
|
||||
local str_start = {}
|
||||
table.insert(str_start, "--" .. boundary .. "\r\n")
|
||||
table.insert(str_start, 'content-disposition: form-data; name="file";')
|
||||
table.insert(str_start, 'filename="' .. "test" ..
|
||||
'"\r\ncontent-type: application/octet-stream\r\n\r\n')
|
||||
str_start = table.concat(str_start)
|
||||
table.insert(sources, ltn12.source.string(str_start))
|
||||
len = len + #str_start
|
||||
|
||||
-- Insert File
|
||||
table.insert(sources, ltn12.source.file(file))
|
||||
len = len + file_size
|
||||
|
||||
-- Insert File End
|
||||
local str_end = "\r\n"
|
||||
table.insert(sources, ltn12.source.string(str_end))
|
||||
len = len + #str_end
|
||||
|
||||
-- Insert Multipart End
|
||||
local str = string.format("--%s--\r\n", boundary)
|
||||
table.insert(sources, ltn12.source.string(str))
|
||||
len = len + #str
|
||||
|
||||
-- Execute Request
|
||||
logger.dbg("SyncNinja: upload_document - Uploading [" .. len .. "]:",
|
||||
filepath)
|
||||
|
||||
local resp = {}
|
||||
local code, headers, status = socket.skip(1, http.request {
|
||||
url = url,
|
||||
method = 'PUT',
|
||||
headers = {
|
||||
["Content-Length"] = len,
|
||||
['Content-Type'] = "multipart/form-data; boundary=" .. boundary,
|
||||
['X-Auth-User'] = username,
|
||||
['X-Auth-Key'] = password
|
||||
},
|
||||
source = ltn12.source.cat(unpack(sources)),
|
||||
sink = ltn12.sink.table(resp)
|
||||
})
|
||||
|
||||
if code ~= 200 then
|
||||
logger.dbg("SyncNinjaClient:upload_document failure:", res)
|
||||
callback(false, res.body)
|
||||
end
|
||||
callback(code == 200, resp)
|
||||
end
|
||||
|
||||
------------------------------------------
|
||||
|
||||
@ -1,7 +1,6 @@
|
||||
local ConfirmBox = require("ui/widget/confirmbox")
|
||||
local DataStorage = require("datastorage")
|
||||
local Device = require("device")
|
||||
local Dispatcher = require("dispatcher")
|
||||
local DocSettings = require("docsettings")
|
||||
local InfoMessage = require("ui/widget/infomessage")
|
||||
local MultiInputDialog = require("ui/widget/multiinputdialog")
|
||||
@ -15,11 +14,6 @@ local _ = require("gettext")
|
||||
local logger = require("logger")
|
||||
local md5 = require("ffi/sha2").md5
|
||||
|
||||
-- TODO:
|
||||
-- - Handle ReadHistory missing files (statistics.sqlite3, bookinfo_cache.sqlite3)
|
||||
-- - Handle document uploads (Manual push only, warning saying this may take awhile)
|
||||
-- - Configure activity bulk size? 1000, 5000, 10000? Separate manual settings to upload ALL?
|
||||
|
||||
------------------------------------------
|
||||
------------ Helper Functions ------------
|
||||
------------------------------------------
|
||||
@ -574,9 +568,10 @@ function SyncNinja:performSync(interactive)
|
||||
self:checkActivity(interactive)
|
||||
self:checkDocuments(interactive)
|
||||
|
||||
-- Notify
|
||||
if interactive == true then
|
||||
UIManager:show(InfoMessage:new{
|
||||
text = _("SyncNinja: Manual Sync Success"),
|
||||
text = _("SyncNinja: Manual Sync Initiated"),
|
||||
timeout = 3
|
||||
})
|
||||
end
|
||||
@ -634,7 +629,6 @@ function SyncNinja:uploadActivity(activity_data, interactive)
|
||||
timeout = 3
|
||||
})
|
||||
end
|
||||
|
||||
return logger.dbg("SyncNinja: uploadActivity Error:", dump(body))
|
||||
end
|
||||
end
|
||||
@ -769,7 +763,6 @@ function SyncNinja:uploadDocumentFiles(doc_metadata, interactive)
|
||||
if self.settings.sync_document_files ~= true then return end
|
||||
if interactive ~= true then return end
|
||||
|
||||
-- API Callback Function
|
||||
local callback_func = function(ok, body)
|
||||
if not ok then
|
||||
UIManager:show(InfoMessage:new{
|
||||
@ -787,24 +780,30 @@ function SyncNinja:uploadDocumentFiles(doc_metadata, interactive)
|
||||
text = _("Uploading Documents - Please Wait...")
|
||||
})
|
||||
|
||||
-- API Client
|
||||
local SyncNinjaClient = require("SyncNinjaClient")
|
||||
local client = SyncNinjaClient:new{
|
||||
custom_url = self.settings.server,
|
||||
service_spec = self.path .. "/api.json"
|
||||
}
|
||||
UIManager:nextTick(function()
|
||||
-- API Client
|
||||
local SyncNinjaClient = require("SyncNinjaClient")
|
||||
local client = SyncNinjaClient:new{
|
||||
custom_url = self.settings.server,
|
||||
service_spec = self.path .. "/api.json"
|
||||
}
|
||||
|
||||
for _, v in pairs(doc_metadata) do
|
||||
if v.filepath ~= nil then
|
||||
-- TODO: Partial File Uploads (Resolve: OOM Issue)
|
||||
local ok, err = pcall(client.upload_document, client,
|
||||
self.settings.username,
|
||||
self.settings.password, v.id, v.filepath,
|
||||
callback_func)
|
||||
for _, v in pairs(doc_metadata) do
|
||||
if v.filepath ~= nil then
|
||||
local ok, err = pcall(client.upload_document, client,
|
||||
self.settings.username,
|
||||
self.settings.password, v.id,
|
||||
v.filepath, callback_func)
|
||||
else
|
||||
logger.dbg("SyncNinja: uploadDocumentFiles - no file for:",
|
||||
v.id)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
UIManager:show(InfoMessage:new{text = _("Uploading Documents Complete")})
|
||||
UIManager:show(InfoMessage:new{
|
||||
text = _("Uploading Documents Complete")
|
||||
})
|
||||
end)
|
||||
end
|
||||
|
||||
UIManager:show(ConfirmBox:new{
|
||||
|
||||
Loading…
Reference in New Issue
Block a user