new OTA mechanism

cmake/CMakeLists.txt

@@ -186,6 +186,7 @@ declare_project(
     md4c
     nanosvg
     utf8proc
+    xxhash
     zlib
     zstd
     SOURCE_DIR thirdparty/kpvcrlib
@@ -196,9 +197,6 @@ set(crengine_BINARY_DIR ${crengine_CMAKE_BINARY_DIR}/build)
 # cpu_features
 declare_project(thirdparty/cpu_features EXCLUDE_FROM_ALL)
 
-# curl
-declare_project(thirdparty/curl DEPENDS libressl zlib EXCLUDE_FROM_ALL)
-
 # czmq
 if(NOT WIN32)
     set(EXCLUDE_FROM_ALL)
@@ -284,7 +282,7 @@ declare_project(thirdparty/leptonica DEPENDS libpng)
 
 # libarchive
 set(DEPENDS zlib zstd)
-if(ANDROID)
+if(NOT (APPLE OR EMULATE_READER))
     list(APPEND DEPENDS xz)
 endif()
 declare_project(thirdparty/libarchive DEPENDS ${DEPENDS})
@@ -411,14 +409,6 @@ declare_project(thirdparty/sdl2 ${EXCLUDE_FROM_ALL})
 # sqlite
 declare_project(thirdparty/sqlite)
 
-# tar
-if(NOT (ANDROID OR APPLE OR EMULATE_READER OR WIN32))
-    set(EXCLUDE_FROM_ALL)
-else()
-    set(EXCLUDE_FROM_ALL EXCLUDE_FROM_ALL)
-endif()
-declare_project(thirdparty/tar ${EXCLUDE_FROM_ALL})
-
 # tesseract
 set(DEPENDS leptonica)
 if(ANDROID)
@@ -432,6 +422,9 @@ declare_project(thirdparty/turbo DEPENDS libressl)
 # utf8proc
 declare_project(thirdparty/utf8proc)
 
+# xxhash
+declare_project(thirdparty/xxhash)
+
 # xz
 declare_project(thirdparty/xz EXCLUDE_FROM_ALL)
 
@@ -441,14 +434,6 @@ declare_project(thirdparty/zlib)
 # zstd
 declare_project(thirdparty/zstd)
 
-# zsync2
-if(CERVANTES OR KINDLE OR KOBO OR POCKETBOOK OR REMARKABLE OR SONY_PRSTUX)
-    set(EXCLUDE_FROM_ALL)
-else()
-    set(EXCLUDE_FROM_ALL EXCLUDE_FROM_ALL)
-endif()
-declare_project(thirdparty/zsync2 DEPENDS curl libressl zlib ${EXCLUDE_FROM_ALL})
-
 # }}}
 
 # TESTSUITE DEPENDENCIES. {{{
@@ -641,6 +626,12 @@ ffi_target(sdl2 SDL2_0_h.lua SDL2_0_decl.c ARGS -d sdl2)
 # utf8proc
 ffi_target(utf8proc utf8proc_h.lua utf8proc_decl.c ARGS -d libutf8proc)
 
+# xxhash
+ffi_target(xxhash xxhash_h.lua xxhash_cdecl.c)
+
+# xz
+ffi_target(xz xz_h.lua xz_cdecl.c)
+
 # zlib
 ffi_target(zlib zlib_h.lua zlib_decl.c ARGS -d zlib)
 

ffi-cdecl/xxhash_cdecl.c

@@ -0,0 +1,14 @@
+#include <xxhash.h>
+
+#include "ffi-cdecl.h"
+
+cdecl_type(XXH3_state_t)
+cdecl_c99_type(XXH64_hash_t, uint64_t)
+
+cdecl_type(XXH_errorcode)
+
+cdecl_func(XXH3_createState)
+cdecl_func(XXH3_freeState)
+cdecl_func(XXH3_64bits_reset)
+cdecl_func(XXH3_64bits_update)
+cdecl_func(XXH3_64bits_digest)

ffi-cdecl/xz_cdecl.c

@@ -0,0 +1,76 @@
+#include <lzma.h>
+
+#include "ffi-cdecl.h"
+
+// base {{{
+
+cdecl_const(LZMA_FILTERS_MAX);
+cdecl_const(LZMA_STREAM_HEADER_SIZE);
+
+cdecl_c99_type(lzma_vli, uint64_t);
+cdecl_type(lzma_allocator);
+cdecl_type(lzma_bool);
+cdecl_type(lzma_reserved_enum);
+cdecl_type(lzma_ret);
+
+// }}}
+
+/// check {{{
+
+cdecl_type(lzma_check);
+
+cdecl_func(lzma_check_size);
+
+/// }}}
+
+/// filter {{{
+
+cdecl_type(lzma_filter);
+
+/// }}}
+
+/// stream {{{
+
+cdecl_type(lzma_stream_flags);
+
+cdecl_func(lzma_stream_flags_compare);
+cdecl_func(lzma_stream_footer_decode);
+cdecl_func(lzma_stream_footer_encode);
+cdecl_func(lzma_stream_header_decode);
+cdecl_func(lzma_stream_header_encode);
+
+// }}}
+
+/// index {{{
+
+cdecl_type(lzma_index);
+cdecl_type(lzma_index_iter);
+cdecl_type(lzma_index_iter_mode);
+
+cdecl_func(lzma_index_append);
+cdecl_func(lzma_index_block_count);
+cdecl_func(lzma_index_buffer_decode);
+cdecl_func(lzma_index_buffer_encode);
+cdecl_func(lzma_index_end);
+cdecl_func(lzma_index_init);
+cdecl_func(lzma_index_iter_init);
+cdecl_func(lzma_index_iter_next);
+cdecl_func(lzma_index_iter_rewind);
+cdecl_func(lzma_index_size);
+cdecl_func(lzma_index_stream_count);
+
+/// }}}
+
+/// block {{{
+
+cdecl_type(lzma_block);
+
+cdecl_func(lzma_block_buffer_decode);
+cdecl_func(lzma_block_compressed_size);
+cdecl_func(lzma_block_header_decode);
+cdecl_func(lzma_block_total_size);
+cdecl_func(lzma_block_unpadded_size);
+
+/// }}}
+
+//  vim: foldmethod=marker foldlevel=0

ffi/downloader.lua

@@ -0,0 +1,87 @@
+local http = require("socket.http")
+local ffi = require("ffi")
+
+local Downloader = {}
+
+function Downloader:new()
+    local o = {}
+    setmetatable(o, self)
+    self.__index = self
+    return o
+end
+
+function Downloader:free()
+    self.ce:free()
+    self.ce = nil
+end
+
+local function merge_ranges(ranges)
+    local new_ranges = {}
+    for i, r in ipairs(ranges) do
+        if #new_ranges > 0 and new_ranges[#new_ranges][2] == r[1] - 1 then
+            new_ranges[#new_ranges][2] = r[2]
+        else
+            table.insert(new_ranges, r)
+        end
+    end
+    return new_ranges
+end
+
+function Downloader:fetch(url, callback, ranges, etag, stats)
+    assert(not (ranges and etag))
+    self.status_code = nil
+    self.etag = nil
+    local ok
+    local sink = function(s)
+        return s and callback(ffi.cast("uint8_t *", s), #s)
+    end
+    local body, status_code, resp_headers, status_line
+    if ranges then
+        ranges = merge_ranges(ranges)
+        local range_support_checked = false
+        local ranges_index = 1
+        repeat
+            body, status_code, resp_headers, status_line = http.request{
+                url = url,
+                headers = { ["Range"] = string.format("bytes=%u-%u", ranges[ranges_index][1], ranges[ranges_index][2]) },
+                sink = sink,
+            }
+            if not body then
+                self.err = status_code
+                return false
+            end
+            if not range_support_checked then
+                if resp_headers["accept-ranges"] ~= "bytes" then
+                    self.err = "server does not support range requests!"
+                    return false
+                end
+                range_support_checked = true
+            end
+            ok = status_code == 206
+            if not ok then
+                self.err = status_line
+                return false
+            end
+            ranges_index = ranges_index + 1
+        until ranges_index > #ranges
+    else
+        body, status_code, resp_headers, status_line = http.request{
+            url = url,
+            headers = etag and { ["If-None-Match"] = etag },
+            sink = sink,
+        }
+        if not body then
+            self.err = status_code
+            return false
+        end
+        self.etag = resp_headers['etag']
+        ok = status_code == 200 or status_code == 304
+        if not ok then
+            self.err = status_line
+        end
+    end
+    self.status_code = status_code
+    return ok
+end
+
+return Downloader

ffi/hashoir.lua

@@ -0,0 +1,41 @@
+local ffi = require("ffi")
+require "ffi/xxhash_h"
+
+local xxhash = ffi.loadlib("xxhash", "0")
+
+local Hashoir = {}
+
+function Hashoir:new()
+    local o = {}
+    setmetatable(o, self)
+    self.__index = self
+    o.hs = ffi.gc(xxhash.XXH3_createState(), xxhash.XXH3_freeState)
+    assert(o.hs ~= nil)
+    xxhash.XXH3_64bits_reset(o.hs)
+    return o
+end
+
+function Hashoir:free()
+    xxhash.XXH3_freeState(ffi.gc(self.hs, nil))
+    self.hs = nil
+end
+
+function Hashoir:reset()
+    xxhash.XXH3_64bits_reset(self.hs)
+    return self
+end
+
+function Hashoir:update(ptr, len)
+    xxhash.XXH3_64bits_update(self.hs, ptr, len)
+    return self
+end
+
+function Hashoir:digest()
+    return xxhash.XXH3_64bits_digest(self.hs)
+end
+
+function Hashoir:hexdigest()
+    return string.format("%016x", xxhash.XXH3_64bits_digest(self.hs))
+end
+
+return Hashoir