AnthoLume/client/syncninja.koplugin/main.lua

local ConfirmBox = require("ui/widget/confirmbox")
local DataStorage = require("datastorage")
local Device = require("device")
local DocSettings = require("docsettings")
local InfoMessage = require("ui/widget/infomessage")
local MultiInputDialog = require("ui/widget/multiinputdialog")
local NetworkMgr = require("ui/network/manager")
local ReadHistory = require("readhistory")
local SQ3 = require("lua-ljsqlite3/init")
local T = require("ffi/util").template
local UIManager = require("ui/uimanager")
local WidgetContainer = require("ui/widget/container/widgetcontainer")
local _ = require("gettext")
local logger = require("logger")
local md5 = require("ffi/sha2").md5
local random = require("random")

------------------------------------------
------------ Helper Functions ------------
------------------------------------------
local function dump(o)
	if type(o) == "table" then
		local s = "{ "
		for k, v in pairs(o) do
			if type(k) ~= "number" then
				k = '"' .. k .. '"'
			end
			s = s .. "[" .. k .. "] = " .. dump(v) .. ","
		end
		return s .. "} "
	else
		return tostring(o)
	end
end

local function validate(entry)
	if not entry then
		return false
	end
	if type(entry) == "string" then
		if entry == "" or not entry:match("%S") then
			return false
		end
	end
	return true
end

local function validateUser(user, pass)
	local error_message = nil
	local user_ok = validate(user)
	local pass_ok = validate(pass)
	if not user_ok and not pass_ok then
		error_message = _("invalid username and password")
	elseif not user_ok then
		error_message = _("invalid username")
	elseif not pass_ok then
		error_message = _("invalid password")
	end

	if not error_message then
		return user_ok and pass_ok
	else
		return user_ok and pass_ok, error_message
	end
end

------------------------------------------
-------------- Plugin Start --------------
------------------------------------------
local MERGE_SETTINGS_IN = "IN"
local MERGE_SETTINGS_OUT = "OUT"

local STATISTICS_ACTIVITY_SINCE_QUERY = [[
    SELECT
	b.md5 AS document,
	psd.start_time AS start_time,
	psd.duration AS duration,
	psd.page AS current_page,
	psd.total_pages
    FROM page_stat_data AS psd
    JOIN book AS b
    ON b.id = psd.id_book
    WHERE start_time > %d
    ORDER BY start_time ASC LIMIT 5000;
]]

local STATISTICS_BOOK_QUERY = [[
    SELECT
	md5,
	title,
	authors,
	series,
	language
      FROM book;
]]

local BOOKINFO_BOOK_QUERY = [[
    SELECT
	(directory || filename) as filepath,
	title,
	authors,
	series,
	series_index,
	language,
	description
    FROM bookinfo;
]]

-- Validate Device ID Exists
if G_reader_settings:hasNot("device_id") then
	G_reader_settings:saveSetting("device_id", random.uuid())
end

-- Define DB Location
local statistics_db = DataStorage:getSettingsDir() .. "/statistics.sqlite3"
local bookinfo_db = DataStorage:getSettingsDir() .. "/bookinfo_cache.sqlite3"

local SyncNinja = WidgetContainer:extend({
	name = "syncninja",
	settings = nil,
	is_doc_only = false,
})

SyncNinja.default_settings = {
	server = nil,
	username = nil,
	password = nil,
	sync_frequency = 30,
	sync_activity = true,
	sync_documents = true,
	sync_document_files = true,
}

function SyncNinja:init()
	logger.dbg("SyncNinja: init")

	-- Instance Specific (Non Interactive)
	self.periodic_push_task = function()
		self:performSync(false)
	end

	-- Load Settings
	self.device_id = G_reader_settings:readSetting("device_id")
	self.settings = G_reader_settings:readSetting("syncninja", self.default_settings)

	-- Register Menu Items
	self.ui.menu:registerToMainMenu(self)

	-- Initial Periodic Push Schedule (5 Minutes)
	self:schedulePeriodicPush(5)
end

------------------------------------------
-------------- UX Functions --------------
------------------------------------------
function SyncNinja:addToMainMenu(menu_items)
	logger.dbg("SyncNinja: addToMainMenu")
	menu_items.syncninja = {
		text = _("Sync Ninja"),
		sorting_hint = "tools",
		sub_item_table = {
			{
				text = _("Sync Server"),
				keep_menu_open = true,
				tap_input_func = function(menu)
					return {
						title = _("Sync server address"),
						input = self.settings.server or "https://",
						type = "text",
						callback = function(input)
							self.settings.server = input ~= "" and input or nil
							if menu then
								menu:updateItems()
							end
						end,
					}
				end,
			},
			{
				text_func = function()
					return self.settings.password and (_("Logout")) or _("Register") .. " / " .. _("Login")
				end,
				enabled_func = function()
					return self.settings.server ~= nil
				end,
				keep_menu_open = true,
				callback_func = function()
					if self.settings.password then
						return function(menu)
							self:logoutUI(menu)
						end
					else
						return function(menu)
							self:loginUI(menu)
						end
					end
				end,
			},
			{
				text = _("Manual Sync"),
				keep_menu_open = true,
				enabled_func = function()
					return self.settings.password ~= nil
						and self.settings.username ~= nil
						and self.settings.server ~= nil
				end,
				callback = function()
					UIManager:unschedule(self.performSync)
					self:performSync(true) -- Interactive
				end,
			},
			{
				text = _("KOSync Auth Merge"),
				sub_item_table = {
					{
						text = _("KOSync Merge In"),
						keep_menu_open = true,
						callback = function()
							self:mergeKOSync(MERGE_SETTINGS_IN)
						end,
					},
					{
						text = _("KOSync Merge Out"),
						keep_menu_open = true,
						callback = function()
							self:mergeKOSync(MERGE_SETTINGS_OUT)
						end,
					},
				},
				separator = true,
			},
			{
				text_func = function()
					return T(_("Sync Frequency (%1 Minutes)"), self.settings.sync_frequency or 30)
				end,
				keep_menu_open = true,
				callback = function(touchmenu_instance)
					local SpinWidget = require("ui/widget/spinwidget")
					local items = SpinWidget:new({
						text = _([[This value determines the cadence at which the syncs will be performed.
If set to 0, periodic sync will be disabled.]]),
						value = self.settings.sync_frequency or 30,
						value_min = 0,
						value_max = 1440,
						value_step = 30,
						value_hold_step = 60,
						ok_text = _("Set"),
						title_text = _("Minutes between syncs"),
						default_value = 30,
						callback = function(spin)
							self.settings.sync_frequency = spin.value > 0 and spin.value or 30
							if touchmenu_instance then
								touchmenu_instance:updateItems()
							end
							self:schedulePeriodicPush()
						end,
					})
					UIManager:show(items)
				end,
			},
			{
				text_func = function()
					return T(
						_("Sync Activity (%1)"),
						self.settings.sync_activity == true and (_("Enabled")) or (_("Disabled"))
					)
				end,
				sub_item_table = {
					{
						text = _("Enabled"),
						checked_func = function()
							return self.settings.sync_activity == true
						end,
						callback = function()
							self.settings.sync_activity = true
						end,
					},
					{
						text = _("Disabled"),
						checked_func = function()
							return self.settings.sync_activity ~= true
						end,
						callback = function()
							self.settings.sync_activity = false
						end,
					},
				},
			},
			{
				text_func = function()
					return T(
						_("Sync Documents (%1)"),
						self.settings.sync_documents == true and (_("Enabled")) or (_("Disabled"))
					)
				end,
				sub_item_table = {
					{
						text = _("Enabled"),
						checked_func = function()
							return self.settings.sync_documents == true
						end,
						callback = function()
							self.settings.sync_documents = true
						end,
					},
					{
						text = _("Disabled"),
						checked_func = function()
							return self.settings.sync_documents ~= true
						end,
						callback = function()
							self.settings.sync_documents = false
						end,
					},
				},
			},
			{
				text_func = function()
					return T(
						_("Sync Document Files (%1)"),
						self.settings.sync_documents == true
								and self.settings.sync_document_files == true
								and (_("Enabled"))
							or (_("Disabled"))
					)
				end,
				enabled_func = function()
					return self.settings.sync_documents == true
				end,
				sub_item_table = {
					{
						text = _("Enabled"),
						checked_func = function()
							return self.settings.sync_document_files == true
						end,
						callback = function()
							self.settings.sync_document_files = true
						end,
					},
					{
						text = _("Disabled"),
						checked_func = function()
							return self.settings.sync_document_files ~= true
						end,
						callback = function()
							self.settings.sync_document_files = false
						end,
					},
				},
			},
		},
	}
end

function SyncNinja:loginUI(menu)
	logger.dbg("SyncNinja: loginUI")
	if NetworkMgr:willRerunWhenOnline(function()
		self:loginUI(menu)
	end) then
		return
	end

	local dialog
	dialog = MultiInputDialog:new({
		title = _("Register/login to SyncNinja server"),
		fields = {
			{ text = self.settings.username, hint = "username" },
			{ hint = "password", text_type = "password" },
		},
		buttons = {
			{
				{
					text = _("Cancel"),
					id = "close",
					callback = function()
						UIManager:close(dialog)
					end,
				},
				{
					text = _("Login"),
					callback = function()
						local username, password = unpack(dialog:getFields())
						local ok, err = validateUser(username, password)
						if not ok then
							UIManager:show(InfoMessage:new({
								text = T(_("Cannot login: %1"), err),
								timeout = 2,
							}))
						else
							UIManager:close(dialog)
							UIManager:scheduleIn(0.5, function()
								self:userLogin(username, password, menu)
							end)
							UIManager:show(InfoMessage:new({
								text = _("Logging in. Please wait…"),
								timeout = 1,
							}))
						end
					end,
				},
				{
					text = _("Register"),
					callback = function()
						local username, password = unpack(dialog:getFields())
						local ok, err = validateUser(username, password)
						if not ok then
							UIManager:show(InfoMessage:new({
								text = T(_("Cannot register: %1"), err),
								timeout = 2,
							}))
						else
							UIManager:close(dialog)
							UIManager:scheduleIn(0.5, function()
								self:userRegister(username, password, menu)
							end)
							UIManager:show(InfoMessage:new({
								text = _("Registering. Please wait…"),
								timeout = 1,
							}))
						end
					end,
				},
			},
		},
	})
	UIManager:show(dialog)
	dialog:onShowKeyboard()
end

function SyncNinja:logoutUI(menu)
	logger.dbg("SyncNinja: logoutUI")
	self.settings.username = nil
	self.settings.password = nil
	if menu then
		menu:updateItems()
	end
	UIManager:unschedule(self.periodic_push_task)
end

function SyncNinja:mergeKOSync(direction)
	logger.dbg("SyncNinja: mergeKOSync")
	local kosync_settings = G_reader_settings:readSetting("kosync")
	if kosync_settings == nil then
		return
	end

	if direction == MERGE_SETTINGS_OUT then
		-- Validate Configured
		if not self.settings.server or not self.settings.username or not self.settings.password then
			return UIManager:show(InfoMessage:new({
				text = _("Error: SyncNinja not configured"),
			}))
		end

		kosync_settings.custom_server = self.settings.server
			.. (self.settings.server:sub(-#"/") == "/" and "api/ko" or "/api/ko")
		kosync_settings.username = self.settings.username
		kosync_settings.userkey = self.settings.password

		UIManager:show(InfoMessage:new({ text = _("Synced to KOSync") }))
	elseif direction == MERGE_SETTINGS_IN then
		-- Validate Configured
		if not kosync_settings.custom_server or not kosync_settings.username or not kosync_settings.userkey then
			return UIManager:show(InfoMessage:new({
				text = _("Error: KOSync not configured"),
			}))
		end

		-- Validate Compatible Server
		if
			kosync_settings.custom_server:sub(-#"/api/ko") ~= "/api/ko"
			and kosync_settings.custom_server:sub(-#"/api/ko/") ~= "/api/ko/"
		then
			return UIManager:show(InfoMessage:new({
				text = _("Error: Configured KOSync server not compatible"),
			}))
		end

		self.settings.server = string.gsub(kosync_settings.custom_server, "/api/ko/?$", "")
		self.settings.username = kosync_settings.username
		self.settings.password = kosync_settings.userkey

		UIManager:show(InfoMessage:new({ text = _("Synced from KOSync") }))
	end
end

------------------------------------------
------------- Login Functions ------------
------------------------------------------
function SyncNinja:userLogin(username, password, menu)
	logger.dbg("SyncNinja: userLogin")
	if not self.settings.server then
		return
	end

	local SyncNinjaClient = require("SyncNinjaClient")
	local client = SyncNinjaClient:new({
		custom_url = self.settings.server,
		service_spec = self.path .. "/api.json",
	})
	Device:setIgnoreInput(true)
	local userkey = md5(password)
	local ok, status, body = pcall(client.authorize, client, username, userkey)
	if not ok then
		if status then
			UIManager:show(InfoMessage:new({
				text = _("An error occurred while logging in:") .. "\n" .. status,
			}))
		else
			UIManager:show(InfoMessage:new({
				text = _("An unknown error occurred while logging in."),
			}))
		end
		Device:setIgnoreInput(false)
		return
	elseif status then
		self.settings.username = username
		self.settings.password = userkey
		if menu then
			menu:updateItems()
		end
		UIManager:show(InfoMessage:new({
			text = _("Logged in to AnthoLume server."),
		}))

		self:schedulePeriodicPush(0)
	else
		logger.dbg("SyncNinja: userLogin Error:", dump(body))
	end
	Device:setIgnoreInput(false)
end

function SyncNinja:userRegister(username, password, menu)
	logger.dbg("SyncNinja: userRegister")
	if not self.settings.server then
		return
	end

	local SyncNinjaClient = require("SyncNinjaClient")
	local client = SyncNinjaClient:new({
		custom_url = self.settings.server,
		service_spec = self.path .. "/api.json",
	})
	-- on Android to avoid ANR (no-op on other platforms)
	Device:setIgnoreInput(true)
	local userkey = md5(password)
	local ok, status, body = pcall(client.register, client, username, userkey)
	if not ok then
		if status then
			UIManager:show(InfoMessage:new({
				text = _("An error occurred while registering:") .. "\n" .. status,
			}))
		else
			UIManager:show(InfoMessage:new({
				text = _("An unknown error occurred while registering."),
			}))
		end
	elseif status then
		self.settings.username = username
		self.settings.password = userkey
		if menu then
			menu:updateItems()
		end
		UIManager:show(InfoMessage:new({
			text = _("Registered to AnthoLume server."),
		}))

		self:schedulePeriodicPush(0)
	else
		UIManager:show(InfoMessage:new({
			text = body and body.message or _("Unknown server error"),
		}))
	end
	Device:setIgnoreInput(false)
end

------------------------------------------
------------- Sync Functions -------------
------------------------------------------
function SyncNinja:schedulePeriodicPush(minutes)
	logger.dbg("SyncNinja: schedulePeriodicPush")

	-- Validate Configured
	if not self.settings then
		return
	end
	if not self.settings.username then
		return
	end
	if not self.settings.password then
		return
	end
	if not self.settings.server then
		return
	end

	-- Unschedule & Schedule
	local sync_frequency = minutes or self.settings.sync_frequency or 30
	UIManager:unschedule(self.periodic_push_task)
	UIManager:scheduleIn(60 * sync_frequency, self.periodic_push_task)
end

function SyncNinja:performSync(interactive)
	logger.dbg("SyncNinja: performSync")

	-- Notify
	if interactive == true then
		UIManager:show(InfoMessage:new({
			text = _("SyncNinja: Manual Sync Initiated"),
			timeout = 3,
		}))
	end

	-- Upload Activity & Check Documents
	self:checkActivity(interactive)
	self:checkDocuments(interactive)

	-- Schedule Push Again
	self:schedulePeriodicPush()
end

function SyncNinja:checkActivity(interactive)
	logger.dbg("SyncNinja: checkActivity")

	-- Ensure Activity Sync Enabled
	if self.settings.sync_activity ~= true then
		return
	end

	-- API Callback Function
	local callback_func = function(ok, body)
		if not ok then
			if interactive == true then
				UIManager:show(InfoMessage:new({
					text = _("SyncNinja: checkActivity Error"),
					timeout = 3,
				}))
			end
			return logger.dbg("SyncNinja: checkActivity Error:", dump(body))
		end

		local last_sync = body.last_sync
		local activity_data = self:getStatisticsActivity(last_sync)

		-- Activity Data Exists
		if not (next(activity_data) == nil) then
			self:uploadActivity(activity_data, interactive)
		end
	end

	-- API Call
	local SyncNinjaClient = require("SyncNinjaClient")
	local client = SyncNinjaClient:new({
		custom_url = self.settings.server,
		service_spec = self.path .. "/api.json",
	})
	local ok, err = pcall(
		client.check_activity,
		client,
		self.settings.username,
		self.settings.password,
		self.device_id,
		Device.model,
		callback_func
	)
end

function SyncNinja:uploadActivity(activity_data, interactive)
	logger.dbg("SyncNinja: uploadActivity")

	-- API Callback Function
	local callback_func = function(ok, body)
		if not ok then
			if interactive == true then
				UIManager:show(InfoMessage:new({
					text = _("SyncNinja: uploadActivity Error"),
					timeout = 3,
				}))
			end
			return logger.dbg("SyncNinja: uploadActivity Error:", dump(body))
		end
	end

	-- API Call
	local SyncNinjaClient = require("SyncNinjaClient")
	local client = SyncNinjaClient:new({
		custom_url = self.settings.server,
		service_spec = self.path .. "/api.json",
	})
	local ok, err = pcall(
		client.add_activity,
		client,
		self.settings.username,
		self.settings.password,
		self.device_id,
		Device.model,
		activity_data,
		callback_func
	)
end

function SyncNinja:checkDocuments(interactive)
	logger.dbg("SyncNinja: checkDocuments")

	-- Ensure Document Sync Enabled
	if self.settings.sync_documents ~= true then
		return
	end

	-- API Request Data
	local doc_metadata = self:getLocalDocumentMetadata()
	local doc_ids = self:getLocalDocumentIDs(doc_metadata)

	-- API Callback Function
	local callback_func = function(ok, body)
		if not ok then
			if interactive == true then
				UIManager:show(InfoMessage:new({
					text = _("SyncNinja: checkDocuments Error"),
					timeout = 3,
				}))
			end
			return logger.dbg("SyncNinja: checkDocuments Error:", dump(body))
		end

		-- Document Metadata Wanted
		if not (next(body.want_metadata) == nil) then
			local hash_want_metadata = {}
			for _, v in pairs(body.want_metadata) do
				hash_want_metadata[v] = true
			end

			local upload_doc_metadata = {}
			for _, v in pairs(doc_metadata) do
				if hash_want_metadata[v.id] == true then
					table.insert(upload_doc_metadata, v)
				end
			end

			self:uploadDocumentMetadata(upload_doc_metadata, interactive)
		end

		-- Document Files Wanted
		if not (next(body.want_files) == nil) then
			local hash_want_files = {}
			for _, v in pairs(body.want_files) do
				hash_want_files[v] = true
			end

			local upload_doc_files = {}
			for _, v in pairs(doc_metadata) do
				if hash_want_files[v.id] == true then
					table.insert(upload_doc_files, v)
				end
			end

			self:uploadDocumentFiles(upload_doc_files, interactive)
		end

		-- Documents Provided
		if not (next(body.give) == nil) then
			self:downloadDocuments(body.give, interactive)
		end
	end

	-- API Call
	local SyncNinjaClient = require("SyncNinjaClient")
	local client = SyncNinjaClient:new({
		custom_url = self.settings.server,
		service_spec = self.path .. "/api.json",
	})
	local ok, err = pcall(
		client.check_documents,
		client,
		self.settings.username,
		self.settings.password,
		self.device_id,
		Device.model,
		doc_ids,
		callback_func
	)
end

function SyncNinja:downloadDocuments(doc_metadata, interactive)
	logger.dbg("SyncNinja: downloadDocuments")

	-- TODO
	--   - OPDS Sufficient?
	--   - Auto Configure OPDS?
end

function SyncNinja:uploadDocumentMetadata(doc_metadata, interactive)
	logger.dbg("SyncNinja: uploadDocumentMetadata")

	-- Ensure Document Sync Enabled
	if self.settings.sync_documents ~= true then
		return
	end

	-- API Callback Function
	local callback_func = function(ok, body)
		if not ok then
			if interactive == true then
				UIManager:show(InfoMessage:new({
					text = _("SyncNinja: uploadDocumentMetadata Error"),
					timeout = 3,
				}))
			end
			return logger.dbg("SyncNinja: uploadDocumentMetadata Error:", dump(body))
		end
	end

	-- API Client
	local SyncNinjaClient = require("SyncNinjaClient")
	local client = SyncNinjaClient:new({
		custom_url = self.settings.server,
		service_spec = self.path .. "/api.json",
	})

	-- API Initial Metadata
	local ok, err =
		pcall(client.add_documents, client, self.settings.username, self.settings.password, doc_metadata, callback_func)
end

function SyncNinja:uploadDocumentFiles(doc_metadata, interactive)
	logger.dbg("SyncNinja: uploadDocumentFiles")

	-- Ensure Document File Sync Enabled
	if self.settings.sync_document_files ~= true then
		return
	end
	if interactive ~= true then
		return
	end

	local callback_func = function(ok, body)
		if not ok then
			UIManager:show(InfoMessage:new({
				text = _("SyncNinja: uploadDocumentFiles Error"),
				timeout = 3,
			}))
			return logger.dbg("SyncNinja: uploadDocumentFiles Error:", dump(body))
		end
	end

	-- API File Upload
	local confirm_upload_callback = function()
		UIManager:show(InfoMessage:new({
			text = _("Uploading Documents - Please Wait..."),
		}))

		UIManager:nextTick(function()
			-- API Client
			local SyncNinjaClient = require("SyncNinjaClient")
			local client = SyncNinjaClient:new({
				custom_url = self.settings.server,
				service_spec = self.path .. "/api.json",
			})

			for _, v in pairs(doc_metadata) do
				if v.filepath ~= nil then
					local ok, err = pcall(
						client.upload_document,
						client,
						self.settings.username,
						self.settings.password,
						v.id,
						v.filepath,
						callback_func
					)
				else
					logger.dbg("SyncNinja: uploadDocumentFiles - no file for:", v.id)
				end
			end

			UIManager:show(InfoMessage:new({
				text = _("Uploading Documents Complete"),
			}))
		end)
	end

	UIManager:show(ConfirmBox:new({
		text = _("Upload documents? This can take awhile."),
		ok_text = _("Yes"),
		ok_callback = confirm_upload_callback,
	}))
end

------------------------------------------
------------ Getter Functions ------------
------------------------------------------
function SyncNinja:getLocalDocumentIDs(doc_metadata)
	logger.dbg("SyncNinja: getLocalDocumentIDs")

	local document_ids = {}

	if doc_metadata == nil then
		doc_metadata = self:getLocalDocumentMetadata()
	end

	for _, v in pairs(doc_metadata) do
		table.insert(document_ids, v.id)
	end

	return document_ids
end

function SyncNinja:getLocalDocumentMetadata()
	logger.dbg("SyncNinja: getLocalDocumentMetadata")

	local all_documents = {}

	local documents_kv = self:getStatisticsBookKV()
	local bookinfo_books = self:getBookInfoBookKV()

	for _, v in pairs(ReadHistory.hist) do
		if DocSettings:hasSidecarFile(v.file) then
			local docsettings = DocSettings:open(v.file)

			-- Ensure Partial MD5 Exists
			local pmd5 = docsettings:readSetting("partial_md5_checksum")
			if not pmd5 then
				pmd5 = self:getPartialMd5(v.file)
				docsettings:saveSetting("partial_md5_checksum", pmd5)
			end

			-- Get Document Props & Ensure Not Nil
			local doc_props = docsettings:readSetting("doc_props") or {}
			local fdoc = bookinfo_books[v.file] or {}

			-- Update or Create
			if documents_kv[pmd5] ~= nil then
				local doc = documents_kv[pmd5]

				-- Merge Statistics, History, and BookInfo
				doc.title = doc.title or doc_props.title or fdoc.title
				doc.author = doc.author or doc_props.authors or fdoc.author
				doc.series = doc.series or doc_props.series or fdoc.series
				doc.lang = doc.lang or doc_props.language or fdoc.lang

				-- Merge History and BookInfo
				doc.series_index = doc_props.series_index or fdoc.series_index
				doc.description = doc_props.description or fdoc.description
				doc.filepath = v.file
			else
				-- Merge History and BookInfo
				documents_kv[pmd5] = {
					title = doc_props.title or fdoc.title,
					author = doc_props.authors or fdoc.author,
					series = doc_props.series or fdoc.series,
					series_index = doc_props.series_index or fdoc.series_index,
					lang = doc_props.language or fdoc.lang,
					description = doc_props.description or fdoc.description,
					filepath = v.file,
				}
			end
		end
	end

	-- Convert KV -> Array
	for pmd5, v in pairs(documents_kv) do
		table.insert(all_documents, {
			id = pmd5,
			title = v.title,
			author = v.author,
			series = v.series,
			series_index = v.series_index,
			lang = v.lang,
			description = v.description,
			filepath = v.filepath,
		})
	end

	return all_documents
end

function SyncNinja:getStatisticsActivity(timestamp)
	logger.dbg("SyncNinja: getStatisticsActivity")

	local all_data = {}
	local conn = SQ3.open(statistics_db)
	local stmt = conn:prepare(string.format(STATISTICS_ACTIVITY_SINCE_QUERY, timestamp))
	local rows = stmt:resultset("i", 5000)
	conn:close()

	-- No Results
	if rows == nil then
		return all_data
	end

	-- Normalize
	for i, v in pairs(rows[1]) do
		table.insert(all_data, {
			document = rows[1][i],
			start_time = tonumber(rows[2][i]),
			duration = tonumber(rows[3][i]),
			page = tonumber(rows[4][i]),
			pages = tonumber(rows[5][i]),
		})
	end

	return all_data
end

-- Returns KEY:VAL (MD5:<TABLE>)
function SyncNinja:getStatisticsBookKV()
	logger.dbg("SyncNinja: getStatisticsBookKV")

	local all_data = {}
	local conn = SQ3.open(statistics_db)
	local stmt = conn:prepare(STATISTICS_BOOK_QUERY)
	local rows = stmt:resultset("i", 1000)
	conn:close()

	-- No Results
	if rows == nil then
		return all_data
	end

	-- Normalize
	for i, v in pairs(rows[1]) do
		local pmd5 = rows[1][i]
		all_data[pmd5] = {
			title = rows[2][i],
			author = rows[3][i],
			series = rows[4][i],
			lang = rows[5][i],
		}
	end

	return all_data
end

-- Returns KEY:VAL (FILEPATH:<TABLE>)
function SyncNinja:getBookInfoBookKV()
	logger.dbg("SyncNinja: getBookInfoBookKV")

	local all_data = {}
	local conn = SQ3.open(bookinfo_db)
	local stmt = conn:prepare(BOOKINFO_BOOK_QUERY)
	local rows = stmt:resultset("i", 1000)
	conn:close()

	-- No Results
	if rows == nil then
		return all_data
	end

	-- Normalize
	for i, v in pairs(rows[1]) do
		filepath = rows[1][i]
		all_data[filepath] = {
			title = rows[2][i],
			author = rows[3][i],
			series = rows[4][i],
			series_index = tonumber(rows[5][i]),
			lang = rows[6][i],
			description = rows[7][i],
		}
	end

	return all_data
end

function SyncNinja:getPartialMd5(file)
	logger.dbg("SyncNinja: getPartialMd5")

	if file == nil then
		return nil
	end
	local bit = require("bit")
	local lshift = bit.lshift
	local step, size = 1024, 1024
	local update = md5()
	local file_handle = io.open(file, "rb")
	if file_handle == nil then
		return nil
	end
	for i = -1, 10 do
		file_handle:seek("set", lshift(step, 2 * i))
		local sample = file_handle:read(size)
		if sample then
			update(sample)
		else
			break
		end
	end
	file_handle:close()
	return update()
end

return SyncNinja