From 889f25752d14d718a56bb7f1aad548b040ddd749 Mon Sep 17 00:00:00 2001 From: Benoit Pierre Date: Sun, 31 Aug 2025 22:05:41 +0200 Subject: [PATCH 01/17] xxhash: add version 0.8.3 --- cmake/CMakeLists.txt | 7 +++++++ ffi-cdecl/xxhash_cdecl.c | 14 +++++++++++++ ffi/loadlib.lua | 1 + ffi/xxhash_h.lua | 17 +++++++++++++++ .../cmake_modules/koreader_targets.cmake | 2 ++ .../koreader_thirdparty_libs.cmake | 4 ++++ thirdparty/kpvcrlib/crengine | 2 +- thirdparty/xxhash/CMakeLists.txt | 21 +++++++++++++++++++ thirdparty/xxhash/overlay/CMakeLists.txt | 12 +++++++++++ 9 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 ffi-cdecl/xxhash_cdecl.c create mode 100644 ffi/xxhash_h.lua create mode 100644 thirdparty/xxhash/CMakeLists.txt create mode 100644 thirdparty/xxhash/overlay/CMakeLists.txt diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 3936f9035..5f98bcdc8 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -186,6 +186,7 @@ declare_project( md4c nanosvg utf8proc + xxhash zlib zstd SOURCE_DIR thirdparty/kpvcrlib @@ -432,6 +433,9 @@ declare_project(thirdparty/turbo DEPENDS libressl) # utf8proc declare_project(thirdparty/utf8proc) +# xxhash +declare_project(thirdparty/xxhash) + # xz declare_project(thirdparty/xz EXCLUDE_FROM_ALL) @@ -641,6 +645,9 @@ ffi_target(sdl2 SDL2_0_h.lua SDL2_0_decl.c ARGS -d sdl2) # utf8proc ffi_target(utf8proc utf8proc_h.lua utf8proc_decl.c ARGS -d libutf8proc) +# xxhash +ffi_target(xxhash xxhash_h.lua xxhash_cdecl.c) + # zlib ffi_target(zlib zlib_h.lua zlib_decl.c ARGS -d zlib) diff --git a/ffi-cdecl/xxhash_cdecl.c b/ffi-cdecl/xxhash_cdecl.c new file mode 100644 index 000000000..742389774 --- /dev/null +++ b/ffi-cdecl/xxhash_cdecl.c @@ -0,0 +1,14 @@ +#include + +#include "ffi-cdecl.h" + +cdecl_type(XXH3_state_t) +cdecl_c99_type(XXH64_hash_t, uint64_t) + +cdecl_type(XXH_errorcode) + +cdecl_func(XXH3_createState) +cdecl_func(XXH3_freeState) +cdecl_func(XXH3_64bits_reset) +cdecl_func(XXH3_64bits_update) +cdecl_func(XXH3_64bits_digest) diff --git a/ffi/loadlib.lua b/ffi/loadlib.lua index 62f4a8a21..db02a1551 100644 --- a/ffi/loadlib.lua +++ b/ffi/loadlib.lua @@ -45,6 +45,7 @@ local monolibtic = { ["webp"] = true, ["webpdemux"] = true, ["wrap-mupdf"] = true, + ["xxhash"] = true, ["z"] = true, ["zmq"] = true, ["zstd"] = true, diff --git a/ffi/xxhash_h.lua b/ffi/xxhash_h.lua new file mode 100644 index 000000000..83ffa94de --- /dev/null +++ b/ffi/xxhash_h.lua @@ -0,0 +1,17 @@ +-- Automatically generated with ffi-cdecl. + +local ffi = require("ffi") + +ffi.cdef[[ +typedef struct XXH3_state_s XXH3_state_t; +typedef uint64_t XXH64_hash_t; +typedef enum { + XXH_OK = 0, + XXH_ERROR = 1, +} XXH_errorcode; +XXH3_state_t *XXH3_createState(void) __attribute__((malloc)); +XXH_errorcode XXH3_freeState(XXH3_state_t *); +XXH_errorcode XXH3_64bits_reset(XXH3_state_t *); +XXH_errorcode XXH3_64bits_update(XXH3_state_t *, const void *, size_t); +XXH64_hash_t XXH3_64bits_digest(const XXH3_state_t *) __attribute__((pure)); +]] diff --git a/thirdparty/cmake_modules/koreader_targets.cmake b/thirdparty/cmake_modules/koreader_targets.cmake index 1a94fcea7..c9dc40295 100644 --- a/thirdparty/cmake_modules/koreader_targets.cmake +++ b/thirdparty/cmake_modules/koreader_targets.cmake @@ -314,6 +314,7 @@ if(MONOLIBTIC) pthread sqlite::sqlite3 turbo::tffi_wrap + xxhash::xxhash zlib::z zstd::zstd SOURCES monolibtic.cpp @@ -359,6 +360,7 @@ if(MONOLIBTIC) tffi_wrap_cdecl turbojpeg_decl utf8proc_decl + xxhash_cdecl zeromq_cdecl zlib_decl zstd_decl diff --git a/thirdparty/cmake_modules/koreader_thirdparty_libs.cmake b/thirdparty/cmake_modules/koreader_thirdparty_libs.cmake index dea138078..d6c18f85b 100644 --- a/thirdparty/cmake_modules/koreader_thirdparty_libs.cmake +++ b/thirdparty/cmake_modules/koreader_thirdparty_libs.cmake @@ -52,6 +52,7 @@ target_link_libraries( md4c::html srell::srell utf8proc::utf8proc + xxhash::xxhash zlib::z zstd::zstd ) @@ -234,6 +235,9 @@ endif() # utf8proc declare_dependency(utf8proc::utf8proc MONOLIBTIC utf8proc) +# xxhash +declare_dependency(xxhash::xxhash MONOLIBTIC xxhash) + # zlib declare_dependency(zlib::z MONOLIBTIC z) diff --git a/thirdparty/kpvcrlib/crengine b/thirdparty/kpvcrlib/crengine index b3d7b3a68..43b741af0 160000 --- a/thirdparty/kpvcrlib/crengine +++ b/thirdparty/kpvcrlib/crengine @@ -1 +1 @@ -Subproject commit b3d7b3a68a6a75a1dda510e7aa8806cd568ddb07 +Subproject commit 43b741af09bc00f010012d5dadcb7f92c52752bf diff --git a/thirdparty/xxhash/CMakeLists.txt b/thirdparty/xxhash/CMakeLists.txt new file mode 100644 index 000000000..5a997d42a --- /dev/null +++ b/thirdparty/xxhash/CMakeLists.txt @@ -0,0 +1,21 @@ +list(APPEND CMAKE_ARGS + -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} + -DBUILD_SHARED_LIBS=$> +) + +list(APPEND BUILD_CMD COMMAND ninja) + +list(APPEND INSTALL_CMD COMMAND ${CMAKE_COMMAND} --install .) + +if(NOT MONOLIBTIC) + append_shared_lib_install_commands(INSTALL_CMD xxhash VERSION 0) +endif() + +external_project( + DOWNLOAD URL 599804eb9555e51c05f1b821f9212a07 + https://github.com/Cyan4973/xxHash/archive/refs/tags/v0.8.3.tar.gz + PATCH_OVERLAY overlay + CMAKE_ARGS ${CMAKE_ARGS} + BUILD_COMMAND ${BUILD_CMD} + INSTALL_COMMAND ${INSTALL_CMD} +) diff --git a/thirdparty/xxhash/overlay/CMakeLists.txt b/thirdparty/xxhash/overlay/CMakeLists.txt new file mode 100644 index 000000000..31918203b --- /dev/null +++ b/thirdparty/xxhash/overlay/CMakeLists.txt @@ -0,0 +1,12 @@ +cmake_minimum_required(VERSION 3.17.5) +project(xxhash LANGUAGES C) + +add_library(xxhash) +set_target_properties(xxhash PROPERTIES SOVERSION 0) +target_sources(xxhash PRIVATE xxhash.c) +if(CMAKE_BUILD_TYPE STREQUAL "Debug") + target_compile_definitions(xxhash PRIVATE XXH_NO_INLINE_HINTS) +endif() + +install(TARGETS xxhash) +install(FILES xxh3.h xxhash.h DESTINATION ${CMAKE_INSTALL_PREFIX}/include) From 7d176eaec46eee3c1407b16da5cf2b9a1eae8461 Mon Sep 17 00:00:00 2001 From: Benoit Pierre Date: Sun, 31 Aug 2025 22:05:42 +0200 Subject: [PATCH 02/17] hashoir: new module for hashing memory Using xxHash's 64bits variant of XXH3. --- ffi/hashoir.lua | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 ffi/hashoir.lua diff --git a/ffi/hashoir.lua b/ffi/hashoir.lua new file mode 100644 index 000000000..180037a17 --- /dev/null +++ b/ffi/hashoir.lua @@ -0,0 +1,41 @@ +local ffi = require("ffi") +require "ffi/xxhash_h" + +local xxhash = ffi.loadlib("xxhash", "0") + +local Hashoir = {} + +function Hashoir:new() + local o = {} + setmetatable(o, self) + self.__index = self + o.hs = ffi.gc(xxhash.XXH3_createState(), xxhash.XXH3_freeState) + assert(o.hs ~= nil) + xxhash.XXH3_64bits_reset(o.hs) + return o +end + +function Hashoir:free() + xxhash.XXH3_freeState(ffi.gc(self.hs, nil)) + self.hs = nil +end + +function Hashoir:reset() + xxhash.XXH3_64bits_reset(self.hs) + return self +end + +function Hashoir:update(ptr, len) + xxhash.XXH3_64bits_update(self.hs, ptr, len) + return self +end + +function Hashoir:digest() + return xxhash.XXH3_64bits_digest(self.hs) +end + +function Hashoir:hexdigest() + return string.format("%016x", xxhash.XXH3_64bits_digest(self.hs)) +end + +return Hashoir From 7580d49d2ba1f0aedcf653442dad22ed314540c5 Mon Sep 17 00:00:00 2001 From: Benoit Pierre Date: Sun, 31 Aug 2025 22:05:43 +0200 Subject: [PATCH 03/17] zstd: build a static library too Needed for unpack. --- thirdparty/zstd/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/thirdparty/zstd/CMakeLists.txt b/thirdparty/zstd/CMakeLists.txt index 2fd11a13c..b0d7bdf4c 100644 --- a/thirdparty/zstd/CMakeLists.txt +++ b/thirdparty/zstd/CMakeLists.txt @@ -7,7 +7,7 @@ list(APPEND CMAKE_ARGS # Project options. -DZSTD_BUILD_PROGRAMS=FALSE -DZSTD_BUILD_SHARED=$> - -DZSTD_BUILD_STATIC=$ + -DZSTD_BUILD_STATIC=TRUE -DZSTD_LEGACY_SUPPORT=FALSE -DZSTD_MULTITHREAD_SUPPORT=FALSE ) From 2a5a3f5feff3ade859d067953d9bee557a12e118 Mon Sep 17 00:00:00 2001 From: Benoit Pierre Date: Sun, 31 Aug 2025 22:05:44 +0200 Subject: [PATCH 04/17] libarchive: enable LZMA support on OTA platforms --- cmake/CMakeLists.txt | 2 +- thirdparty/cmake_modules/koreader_thirdparty_libs.cmake | 2 +- thirdparty/libarchive/CMakeLists.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 5f98bcdc8..0ee87b612 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -285,7 +285,7 @@ declare_project(thirdparty/leptonica DEPENDS libpng) # libarchive set(DEPENDS zlib zstd) -if(ANDROID) +if(NOT (APPLE OR EMULATE_READER)) list(APPEND DEPENDS xz) endif() declare_project(thirdparty/libarchive DEPENDS ${DEPENDS}) diff --git a/thirdparty/cmake_modules/koreader_thirdparty_libs.cmake b/thirdparty/cmake_modules/koreader_thirdparty_libs.cmake index d6c18f85b..a744441ec 100644 --- a/thirdparty/cmake_modules/koreader_thirdparty_libs.cmake +++ b/thirdparty/cmake_modules/koreader_thirdparty_libs.cmake @@ -189,7 +189,7 @@ if(ANDROID) list(APPEND SYS_LIBS log) endif() set(MONO_LIBS archive freetype harfbuzz jpeg webp webpdemux z) -if(ANDROID) +if(NOT (APPLE OR EMULATE_READER)) list(APPEND STATIC_LIBS lzma) endif() declare_dependency( diff --git a/thirdparty/libarchive/CMakeLists.txt b/thirdparty/libarchive/CMakeLists.txt index 062162368..25c6b26ed 100644 --- a/thirdparty/libarchive/CMakeLists.txt +++ b/thirdparty/libarchive/CMakeLists.txt @@ -17,7 +17,7 @@ list(APPEND CMAKE_ARGS -DENABLE_LIBB2=FALSE -DENABLE_LIBXML2=FALSE -DENABLE_LZ4=FALSE - -DENABLE_LZMA=$ + -DENABLE_LZMA=$,$>> -DENABLE_OPENSSL=FALSE -DENABLE_PCRE2POSIX=FALSE -DENABLE_PCREPOSIX=FALSE From 93ea3ff2c42be25f9b3a8252a616b4d394c13d73 Mon Sep 17 00:00:00 2001 From: Benoit Pierre Date: Sun, 31 Aug 2025 22:05:45 +0200 Subject: [PATCH 05/17] unpack: add tool A minimal libarchive based archive extractor with support for TAR / ZIP archives, and LZMA / GZip / ZSTD / XZ compression. --- .../cmake_modules/koreader_targets.cmake | 13 + .../koreader_thirdparty_libs.cmake | 6 + unpack.c | 379 ++++++++++++++++++ 3 files changed, 398 insertions(+) create mode 100644 unpack.c diff --git a/thirdparty/cmake_modules/koreader_targets.cmake b/thirdparty/cmake_modules/koreader_targets.cmake index c9dc40295..b1a4c0e95 100644 --- a/thirdparty/cmake_modules/koreader_targets.cmake +++ b/thirdparty/cmake_modules/koreader_targets.cmake @@ -270,6 +270,19 @@ declare_koreader_target( SOURCES button-listen.c ) +# unpack +if(NOT (ANDROID OR APPLE OR EMULATE_READER)) + set(EXCLUDE_FROM_ALL) +else() + set(EXCLUDE_FROM_ALL EXCLUDE_FROM_ALL) +endif() +declare_koreader_target( + unpack TYPE executable + DEPENDS libarchive::libarchive_static xz::lzma_static zlib::z_static zstd::zstd_static m + ${EXCLUDE_FROM_ALL} + SOURCES unpack.c +) + # }}} # MONOLIBTIC. {{{ diff --git a/thirdparty/cmake_modules/koreader_thirdparty_libs.cmake b/thirdparty/cmake_modules/koreader_thirdparty_libs.cmake index a744441ec..210b6ab24 100644 --- a/thirdparty/cmake_modules/koreader_thirdparty_libs.cmake +++ b/thirdparty/cmake_modules/koreader_thirdparty_libs.cmake @@ -90,6 +90,7 @@ declare_dependency(harfbuzz::harfbuzz INCLUDES freetype2 harfbuzz MONOLIBTIC har # libarchive declare_dependency(libarchive::libarchive MONOLIBTIC archive) +declare_dependency(libarchive::libarchive_static STATIC archive) # leptonica declare_dependency(leptonica::leptonica INCLUDES leptonica MONOLIBTIC leptonica) @@ -238,8 +239,13 @@ declare_dependency(utf8proc::utf8proc MONOLIBTIC utf8proc) # xxhash declare_dependency(xxhash::xxhash MONOLIBTIC xxhash) +# xz +declare_dependency(xz::lzma_static STATIC lzma) + # zlib declare_dependency(zlib::z MONOLIBTIC z) +declare_dependency(zlib::z_static STATIC z) # zstd declare_dependency(zstd::zstd MONOLIBTIC zstd) +declare_dependency(zstd::zstd_static STATIC zstd) diff --git a/unpack.c b/unpack.c new file mode 100644 index 000000000..a08e6d224 --- /dev/null +++ b/unpack.c @@ -0,0 +1,379 @@ +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + + +static void _archive_error(struct archive *archive) { + const char *err = archive_error_string(archive); + fprintf(stderr, "ERROR: %s\n", err ? err : "Unknown error"); +} + +static int _gzip_size(struct archive *archive, const char *path, la_int64_t *ps) { + uint32_t size; + int err; + int fd; + + err = ARCHIVE_FAILED; + + fd = open(path, O_RDONLY); + if (fd < 0) { + archive_set_error(archive, ARCHIVE_FAILED, "%s", strerror(errno)); + goto end; + } + + if (lseek(fd, -4, SEEK_END) <= 0 || read(fd, &size, 4) != 4) { + archive_set_error(archive, ARCHIVE_FAILED, "Failed to read GZIP footer"); + goto end; + } + + *ps = size; + err = ARCHIVE_OK; + +end: + if (fd != -1) + close(fd); + + return err; +} + +static int _xz_size(struct archive *archive, const char *path, la_int64_t *ps) { + uint8_t stream_footer[LZMA_STREAM_HEADER_SIZE]; + lzma_stream_flags stream_flags; + uint8_t *index_buf; + lzma_index *index; + lzma_vli size; + int err; + int fd; + + index = NULL; + index_buf = NULL; + err = ARCHIVE_FAILED; + + fd = open(path, O_RDONLY); + if (fd < 0) { + archive_set_error(archive, ARCHIVE_FAILED, "%s", strerror(errno)); + goto end; + } + + if (lseek(fd, -LZMA_STREAM_HEADER_SIZE, SEEK_END) <= 0 || + read(fd, stream_footer, LZMA_STREAM_HEADER_SIZE) != LZMA_STREAM_HEADER_SIZE || + lzma_stream_footer_decode(&stream_flags, stream_footer) != LZMA_OK) { + archive_set_error(archive, ARCHIVE_FAILED, "Failed to read XZ footer"); + goto end; + } + + index_buf = malloc(stream_flags.backward_size); + if (!index_buf) { + archive_set_error(archive, ARCHIVE_FAILED, "%s", strerror(errno)); + goto end; + } + + { + uint64_t memlimit = INT_MAX; + size_t pos = 0; + + if (lseek(fd, -LZMA_STREAM_HEADER_SIZE - stream_flags.backward_size, SEEK_END) <= 0 || + read(fd, index_buf, stream_flags.backward_size) != stream_flags.backward_size || + lzma_index_buffer_decode(&index, &memlimit, NULL, index_buf, &pos, stream_flags.backward_size) != LZMA_OK) { + archive_set_error(archive, ARCHIVE_FAILED, "Failed to read XZ index"); + goto end; + } + } + + { + lzma_index_iter index_iter; + + size = 2 * LZMA_STREAM_HEADER_SIZE + stream_flags.backward_size; + + lzma_index_iter_init(&index_iter, index); + while (!lzma_index_iter_next(&index_iter, LZMA_INDEX_ITER_NONEMPTY_BLOCK)) + size += index_iter.block.total_size; + } + + *ps = size; + err = ARCHIVE_OK; + +end: + if (index) + lzma_index_end(index, NULL); + if (index_buf) + free(index_buf); + if (fd != -1) + close(fd); + + return err; +} + +static int _archive_total_size(struct archive *archive, const char *path, la_int64_t *ps, struct archive_entry **pe) { + enum { + SIZE_FROM_FILE_SIZE, + SIZE_FROM_GZIP_FOOTER, + SIZE_FROM_XZ_INDEX, + } mode; + struct archive_entry *entry; + int err; + + if (!path) { + archive_set_error(archive, ARCHIVE_FAILED, "Cannot determize size of standard input"); + return ARCHIVE_FAILED; + } + + err = archive_read_next_header(archive, &entry); + if (err != ARCHIVE_OK) + return err; + + switch (archive_format(archive)) { + case ARCHIVE_FORMAT_TAR: + case ARCHIVE_FORMAT_TAR_USTAR: + case ARCHIVE_FORMAT_TAR_PAX_INTERCHANGE: + case ARCHIVE_FORMAT_TAR_PAX_RESTRICTED: + case ARCHIVE_FORMAT_TAR_GNUTAR: + { + switch (archive_filter_code(archive, 0)) { + case ARCHIVE_FILTER_NONE: + mode = SIZE_FROM_FILE_SIZE; + break; + case ARCHIVE_FILTER_GZIP: + mode = SIZE_FROM_GZIP_FOOTER; + break; + case ARCHIVE_FILTER_XZ: + mode = SIZE_FROM_XZ_INDEX; + break; + default: + archive_set_error(archive, ARCHIVE_FAILED, "Unsupported TAR compression filter"); + return ARCHIVE_FAILED; + } + } + break; + case ARCHIVE_FORMAT_ZIP: + mode = SIZE_FROM_FILE_SIZE; + break; + default: + archive_set_error(archive, ARCHIVE_FAILED, "Unrecognized or unsupported archive format"); + return ARCHIVE_FAILED; + } + + switch (mode) { + case SIZE_FROM_FILE_SIZE: + { + struct stat st; + + if (stat(path, &st)) { + archive_set_error(archive, ARCHIVE_FAILED, "%s", strerror(errno)); + return ARCHIVE_FAILED; + } + + *ps = st.st_size; + } + break; + case SIZE_FROM_GZIP_FOOTER: + err = _gzip_size(archive, path, ps); + break; + case SIZE_FROM_XZ_INDEX: + err = _xz_size(archive, path, ps); + break; + } + + *pe = entry; + + return ARCHIVE_OK; +} + +static int _archive_list(struct archive *archive) { + struct archive_entry *entry; + int err; + + for (;;) { + err = archive_read_next_header(archive, &entry); + if (err == ARCHIVE_EOF) + return ARCHIVE_OK; + if (err != ARCHIVE_OK) + return err; + const char *path = archive_entry_pathname(entry); + write(1, path, strlen(path)); + write(1, "\n", 1); + } +} + +struct progress_ctx { + struct archive *archive; + la_int64_t progress; + double percent; + double size; + char size_unit; + la_int64_t total_size; +}; + +#define KILO (1000) +#define MEGA (KILO * 1000) +#define GIGA (MEGA * 1000) + +static void print_progress(struct progress_ctx *ctx, la_int64_t progress) { + int line_length; + char line[16]; + + if (progress == ctx->progress) + return; + ctx->progress = progress; + + if (ctx->total_size) { + double percent = floor((double)progress / ctx->total_size * 100.0); + + if (percent == ctx->percent) + return; + ctx->percent = percent; + + line_length = snprintf(line, sizeof (line), "\r%3.0f%%", percent); + } else { + double size; + char size_unit; + + if (progress >= GIGA) { + size = (double)progress / GIGA; + size_unit = 'G'; + } else if (progress >= MEGA) { + size = (double)progress / MEGA; + size_unit = 'M'; + } else { + size = (double)progress / KILO; + size_unit = 'K'; + } + + if (size_unit == ctx->size_unit && (size - ctx->size) < 0.1) + return; + ctx->size_unit = size_unit; + ctx->size = size; + + line_length = snprintf(line, sizeof (line), "\r%5.1f%c", size, size_unit); + } + + write(1, line, line_length); +} + +static void extract_progress(void *u) { + struct progress_ctx *ctx = u; + + print_progress(ctx, archive_filter_bytes(ctx->archive, 0)); +} + +static int _archive_extract(struct archive *archive, const char *path, int progress) { + struct progress_ctx progress_ctx; + struct archive *o; + struct archive_entry *entry; + int err; + + o = archive_write_disk_new(); + + if (progress) { + progress_ctx.archive = archive; + progress_ctx.progress = 0; + progress_ctx.percent = -1; + progress_ctx.size_unit = '\0'; + progress_ctx.size = 0; + progress_ctx.total_size = 0; + + // Disable output buffering. + setvbuf(stdout, NULL, _IONBF, 0); + + archive_read_extract_set_progress_callback(archive, extract_progress, &progress_ctx); + if (path) { + err = _archive_total_size(archive, path, &progress_ctx.total_size, &entry); + if (err != ARCHIVE_OK) + return err; + goto direct_to_extract; + } + } + + for (;;) { + err = archive_read_next_header(archive, &entry); + if (err == ARCHIVE_EOF) { + if (progress) { + if (progress_ctx.total_size) + print_progress(&progress_ctx, progress_ctx.total_size); + write(1, "\n", 1); + } + return ARCHIVE_OK; + } + if (err != ARCHIVE_OK) + return err; +direct_to_extract: + err = archive_read_extract2(archive, entry, o); + if (err != ARCHIVE_OK) + return err; + } +} + +static void usage(const char *arg0) { + fprintf(stderr, + "%s [-l|-x|-X] ARCHIVE\n" + "\n" + " -l list archive contents\n" + " -x extract archive\n" + " -X extract archive with progress counter\n" + , arg0); +} + +int main(int argc, char **argv) { + enum { + MODE_EXTRACT, + MODE_EXTRACT_WITH_PROGRESS, + MODE_LIST, + } mode; + struct archive *archive; + const char *path; + int err; + + if (argc != 3) { + usage(argv[0]); + return 1; + } + + if (0 == strcmp(argv[1], "-l")) + mode = MODE_LIST; + else if (0 == strcmp(argv[1], "-x")) + mode = MODE_EXTRACT; + else if (0 == strcmp(argv[1], "-X")) + mode = MODE_EXTRACT_WITH_PROGRESS; + else { + usage(argv[0]); + return strcmp(argv[1], "-h") ? 1 : 0; + } + + archive = archive_read_new(); + archive_read_support_filter_gzip(archive); + archive_read_support_filter_none(archive); + archive_read_support_filter_xz(archive); + archive_read_support_format_tar(archive); + archive_read_support_format_zip(archive); + + path = strcmp(argv[2], "-") ? argv[2] : NULL; + + err = archive_read_open_filename(archive, path, 10240); + if (err == ARCHIVE_OK) { + switch (mode) { + case MODE_EXTRACT: + case MODE_EXTRACT_WITH_PROGRESS: + err = _archive_extract(archive, path, mode == MODE_EXTRACT_WITH_PROGRESS); + break; + case MODE_LIST: + err = _archive_list(archive); + break; + } + } + + if (err != ARCHIVE_OK) + _archive_error(archive); + + archive_read_close(archive); + archive_read_free(archive); + + return err != ARCHIVE_OK ? 2 : 0; +} From e47f9733e09075ae587c734ce9ead67cebd53e4a Mon Sep 17 00:00:00 2001 From: Benoit Pierre Date: Sun, 31 Aug 2025 22:05:46 +0200 Subject: [PATCH 06/17] tar: drop project - archive creation: not need by the new OTA mechanism anyway - extraction: replaced by unpack --- cmake/CMakeLists.txt | 8 - thirdparty/tar/CMakeLists.txt | 62 - ...31-revert-wordsplit-for-android-glob.patch | 2635 ----------------- .../tar/tar-1.33-remove-o_path-usage.patch | 24 - 4 files changed, 2729 deletions(-) delete mode 100644 thirdparty/tar/CMakeLists.txt delete mode 100644 thirdparty/tar/tar-1.31-revert-wordsplit-for-android-glob.patch delete mode 100644 thirdparty/tar/tar-1.33-remove-o_path-usage.patch diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 0ee87b612..6229956c4 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -412,14 +412,6 @@ declare_project(thirdparty/sdl2 ${EXCLUDE_FROM_ALL}) # sqlite declare_project(thirdparty/sqlite) -# tar -if(NOT (ANDROID OR APPLE OR EMULATE_READER OR WIN32)) - set(EXCLUDE_FROM_ALL) -else() - set(EXCLUDE_FROM_ALL EXCLUDE_FROM_ALL) -endif() -declare_project(thirdparty/tar ${EXCLUDE_FROM_ALL}) - # tesseract set(DEPENDS leptonica) if(ANDROID) diff --git a/thirdparty/tar/CMakeLists.txt b/thirdparty/tar/CMakeLists.txt deleted file mode 100644 index ab17d8a2c..000000000 --- a/thirdparty/tar/CMakeLists.txt +++ /dev/null @@ -1,62 +0,0 @@ -# O_PATH may be defined in our ToolChains, but older devices actually run kernels where it's not entirely usable... -# (Usage in tar via gnulib's https://git.savannah.gnu.org/cgit/gnulib.git/log/lib/fchmodat.c) -if(NOT EMULATE_READER) - list(APPEND PATCH_FILES tar-1.33-remove-o_path-usage.patch) -endif() - -# The wordsplit changes introduced in 1.31 require glob.h, which is not present on Androdi before ABI 28 (c.f., #787) -if(ANDROID) - list(APPEND PATCH_FILES tar-1.31-revert-wordsplit-for-android-glob.patch) -endif() - -# list(APPEND PATCH_CMD COMMAND touch ABOUT-NLS po/Makevars.template) -# list(APPEND PATCH_CMD COMMAND env AUTOPOINT=true ./bootstrap --skip-po) - -set(CPPFLAGS "-DHAVE_MKFIFO=1") - -if(ANDROID) - set(LIBS -static) -elseif(APPLE OR WIN32) - set(LIBS) -else() - set(LIBS -lrt) -endif() - -list(APPEND CFG_CMD COMMAND env) -# Avoid pulling > GLIBC_2.4 symbols on crappy platforms -if(LEGACY OR POCKETBOOK) - list(APPEND CFG_CMD ac_cv_func_utimensat=no ac_cv_func_futimens=no) -endif() -append_autotools_vars(CFG_CMD) -list(APPEND CFG_CMD - ${SOURCE_DIR}/configure --host=${CHOST} - --disable-acl - --disable-gcc-warnings - --disable-nls - --without-posix-acls - --without-selinux - --without-xattrs -) -if(LEGACY OR POCKETBOOK) - list(APPEND CFG_CMD --disable-largefile) -endif() - -if(LEGACY OR POCKETBOOK) - # Forcibly disable FORTIFY on legacy devices... - list(APPEND CFG_CMD COMMAND ${ISED} "s/# define _FORTIFY_SOURCE 2/#undef _FORTIFY_SOURCE/" config.h) -endif() - -list(APPEND BUILD_CMD COMMAND make) - -append_binary_install_command(INSTALL_CMD src/tar) - -external_project( - DOWNLOAD URL 9d5949e4c2d9665546ac65dafc0e726a - http://ftpmirror.gnu.org/tar/tar-1.34.tar.gz - https://ftp.wayne.edu/gnu/tar/tar-1.34.tar.gz - http://ftp.gnu.org/pub/gnu/tar/tar-1.34.tar.gz - PATCH_FILES ${PATCH_FILES} - CONFIGURE_COMMAND ${CFG_CMD} - BUILD_COMMAND ${BUILD_CMD} - INSTALL_COMMAND ${INSTALL_CMD} -) diff --git a/thirdparty/tar/tar-1.31-revert-wordsplit-for-android-glob.patch b/thirdparty/tar/tar-1.31-revert-wordsplit-for-android-glob.patch deleted file mode 100644 index c6c6f53b5..000000000 --- a/thirdparty/tar/tar-1.31-revert-wordsplit-for-android-glob.patch +++ /dev/null @@ -1,2635 +0,0 @@ -diff --git a/lib/wordsplit.c b/lib/wordsplit.c -index 661a4f8..f2ecada 100644 ---- a/lib/wordsplit.c -+++ b/lib/wordsplit.c -@@ -1,5 +1,5 @@ - /* wordsplit - a word splitter -- Copyright (C) 2009-2018 Sergey Poznyakoff -+ Copyright (C) 2009-2014, 2016-2017 Free Software Foundation, Inc. - - This program is free software; you can redistribute it and/or modify it - under the terms of the GNU General Public License as published by the -@@ -12,7 +12,10 @@ - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along -- with this program. If not, see . */ -+ with this program. If not, see . -+ -+ Written by Sergey Poznyakoff -+*/ - - #ifdef HAVE_CONFIG_H - # include -@@ -25,8 +28,6 @@ - #include - #include - #include --#include --#include - - #if ENABLE_NLS - # include -@@ -36,14 +37,6 @@ - #define _(msgid) gettext (msgid) - #define N_(msgid) msgid - --#ifndef FALLTHROUGH --# if __GNUC__ < 7 --# define FALLTHROUGH ((void) 0) --# else --# define FALLTHROUGH __attribute__ ((__fallthrough__)) --# endif --#endif -- - #include - - #define ISWS(c) ((c)==' '||(c)=='\t'||(c)=='\n') -@@ -58,19 +51,13 @@ - #define ISALNUM(c) (ISALPHA(c) || ISDIGIT(c)) - #define ISPRINT(c) (' ' <= ((unsigned) (c)) && ((unsigned) (c)) <= 127) - --#define ISVARBEG(c) (ISALPHA(c) || c == '_') --#define ISVARCHR(c) (ISALNUM(c) || c == '_') -- --#define WSP_RETURN_DELIMS(wsp) \ -- ((wsp)->ws_flags & WRDSF_RETURN_DELIMS || ((wsp)->ws_options & WRDSO_MAXWORDS)) -- - #define ALLOC_INIT 128 - #define ALLOC_INCR 128 - - static void - _wsplt_alloc_die (struct wordsplit *wsp) - { -- wsp->ws_error ("%s", _("memory exhausted")); -+ wsp->ws_error (_("memory exhausted")); - abort (); - } - -@@ -87,15 +74,6 @@ _wsplt_error (const char *fmt, ...) - - static void wordsplit_free_nodes (struct wordsplit *); - --static int --_wsplt_seterr (struct wordsplit *wsp, int ec) --{ -- wsp->ws_errno = ec; -- if (wsp->ws_flags & WRDSF_SHOWERR) -- wordsplit_perror (wsp); -- return ec; --} -- - static int - _wsplt_nomem (struct wordsplit *wsp) - { -@@ -111,84 +89,6 @@ _wsplt_nomem (struct wordsplit *wsp) - return wsp->ws_errno; - } - --static int wordsplit_run (const char *command, size_t length, -- struct wordsplit *wsp, -- unsigned flags, int lvl); -- --static int wordsplit_init (struct wordsplit *wsp, const char *input, size_t len, -- unsigned flags); --static int wordsplit_process_list (struct wordsplit *wsp, size_t start); --static int wordsplit_finish (struct wordsplit *wsp); -- --static int --_wsplt_subsplit (struct wordsplit *wsp, struct wordsplit *wss, -- char const *str, int len, -- unsigned flags, int finalize) --{ -- int rc; -- -- wss->ws_delim = wsp->ws_delim; -- wss->ws_debug = wsp->ws_debug; -- wss->ws_error = wsp->ws_error; -- wss->ws_alloc_die = wsp->ws_alloc_die; -- -- if (!(flags & WRDSF_NOVAR)) -- { -- wss->ws_env = wsp->ws_env; -- wss->ws_getvar = wsp->ws_getvar; -- flags |= wsp->ws_flags & (WRDSF_ENV | WRDSF_ENV_KV | WRDSF_GETVAR); -- } -- if (!(flags & WRDSF_NOCMD)) -- { -- wss->ws_command = wsp->ws_command; -- } -- -- if ((flags & (WRDSF_NOVAR|WRDSF_NOCMD)) != (WRDSF_NOVAR|WRDSF_NOCMD)) -- { -- wss->ws_closure = wsp->ws_closure; -- flags |= wsp->ws_flags & WRDSF_CLOSURE; -- } -- -- wss->ws_options = wsp->ws_options; -- -- flags |= WRDSF_DELIM -- | WRDSF_ALLOC_DIE -- | WRDSF_ERROR -- | WRDSF_DEBUG -- | (wsp->ws_flags & (WRDSF_SHOWDBG | WRDSF_SHOWERR | WRDSF_OPTIONS)); -- -- rc = wordsplit_init (wss, str, len, flags); -- if (rc) -- return rc; -- wss->ws_lvl = wsp->ws_lvl + 1; -- rc = wordsplit_process_list (wss, 0); -- if (rc) -- { -- wordsplit_free_nodes (wss); -- return rc; -- } -- if (finalize) -- { -- rc = wordsplit_finish (wss); -- wordsplit_free_nodes (wss); -- } -- return rc; --} -- --static void --_wsplt_seterr_sub (struct wordsplit *wsp, struct wordsplit *wss) --{ -- if (wsp->ws_errno == WRDSE_USERERR) -- free (wsp->ws_usererr); -- wsp->ws_errno = wss->ws_errno; -- if (wss->ws_errno == WRDSE_USERERR) -- { -- wsp->ws_usererr = wss->ws_usererr; -- wss->ws_errno = WRDSE_EOF; -- wss->ws_usererr = NULL; -- } --} -- - static void - wordsplit_init0 (struct wordsplit *wsp) - { -@@ -196,7 +96,6 @@ wordsplit_init0 (struct wordsplit *wsp) - { - if (!(wsp->ws_flags & WRDSF_APPEND)) - wordsplit_free_words (wsp); -- wordsplit_clearerr (wsp); - } - else - { -@@ -206,13 +105,12 @@ wordsplit_init0 (struct wordsplit *wsp) - } - - wsp->ws_errno = 0; -+ wsp->ws_head = wsp->ws_tail = NULL; - } - --char wordsplit_c_escape_tab[] = "\\\\\"\"a\ab\bf\fn\nr\rt\tv\v"; -- - static int - wordsplit_init (struct wordsplit *wsp, const char *input, size_t len, -- unsigned flags) -+ int flags) - { - wsp->ws_flags = flags; - -@@ -221,21 +119,23 @@ wordsplit_init (struct wordsplit *wsp, const char *input, size_t len, - if (!(wsp->ws_flags & WRDSF_ERROR)) - wsp->ws_error = _wsplt_error; - -- if (!(wsp->ws_flags & WRDSF_NOVAR)) -+ if (!(wsp->ws_flags & WRDSF_NOVAR) -+ && !(wsp->ws_flags & (WRDSF_ENV | WRDSF_GETVAR))) - { -- /* These will be initialized on first variable assignment */ -- wsp->ws_envidx = wsp->ws_envsiz = 0; -- wsp->ws_envbuf = NULL; -+ errno = EINVAL; -+ wsp->ws_errno = WRDSE_USAGE; -+ if (wsp->ws_flags & WRDSF_SHOWERR) -+ wordsplit_perror (wsp); -+ return wsp->ws_errno; - } - - if (!(wsp->ws_flags & WRDSF_NOCMD)) - { -- if (!wsp->ws_command) -- { -- _wsplt_seterr (wsp, WRDSE_USAGE); -- errno = EINVAL; -- return wsp->ws_errno; -- } -+ errno = EINVAL; -+ wsp->ws_errno = WRDSE_NOSUPP; -+ if (wsp->ws_flags & WRDSF_SHOWERR) -+ wordsplit_perror (wsp); -+ return wsp->ws_errno; - } - - if (wsp->ws_flags & WRDSF_SHOWDBG) -@@ -266,42 +166,10 @@ wordsplit_init (struct wordsplit *wsp, const char *input, size_t len, - if (!(wsp->ws_flags & WRDSF_CLOSURE)) - wsp->ws_closure = NULL; - -- if (!(wsp->ws_flags & WRDSF_OPTIONS)) -- wsp->ws_options = 0; -- -- if (wsp->ws_flags & WRDSF_ESCAPE) -- { -- if (!wsp->ws_escape[WRDSX_WORD]) -- wsp->ws_escape[WRDSX_WORD] = ""; -- if (!wsp->ws_escape[WRDSX_QUOTE]) -- wsp->ws_escape[WRDSX_QUOTE] = ""; -- } -- else -- { -- if (wsp->ws_flags & WRDSF_CESCAPES) -- { -- wsp->ws_escape[WRDSX_WORD] = wordsplit_c_escape_tab; -- wsp->ws_escape[WRDSX_QUOTE] = wordsplit_c_escape_tab; -- wsp->ws_options |= WRDSO_OESC_QUOTE | WRDSO_OESC_WORD -- | WRDSO_XESC_QUOTE | WRDSO_XESC_WORD; -- } -- else -- { -- wsp->ws_escape[WRDSX_WORD] = ""; -- wsp->ws_escape[WRDSX_QUOTE] = "\\\\\"\""; -- wsp->ws_options |= WRDSO_BSKEEP_QUOTE; -- } -- } -- - wsp->ws_endp = 0; -- wsp->ws_wordi = 0; - -- if (wsp->ws_flags & WRDSF_REUSE) -- wordsplit_free_nodes (wsp); -- wsp->ws_head = wsp->ws_tail = NULL; -- - wordsplit_init0 (wsp); -- -+ - return 0; - } - -@@ -344,7 +212,6 @@ alloc_space (struct wordsplit *wsp, size_t count) - #define _WSNF_NOEXPAND 0x08 /* text is not subject to expansion */ - #define _WSNF_JOIN 0x10 /* node must be joined with the next node */ - #define _WSNF_SEXP 0x20 /* is a sed expression */ --#define _WSNF_DELIM 0x40 /* node is a delimiter */ - - #define _WSNF_EMPTYOK 0x0100 /* special flag indicating that - wordsplit_add_segm must add the -@@ -367,9 +234,9 @@ struct wordsplit_node - }; - - static const char * --wsnode_flagstr (unsigned flags) -+wsnode_flagstr (int flags) - { -- static char retbuf[7]; -+ static char retbuf[6]; - char *p = retbuf; - - if (flags & _WSNF_WORD) -@@ -394,10 +261,6 @@ wsnode_flagstr (unsigned flags) - *p++ = 's'; - else - *p++ = '-'; -- if (flags & _WSNF_DELIM) -- *p++ = 'd'; -- else -- *p++ = '-'; - *p = 0; - return retbuf; - } -@@ -478,14 +341,6 @@ wsnode_remove (struct wordsplit *wsp, struct wordsplit_node *node) - node->next = node->prev = NULL; - } - --static struct wordsplit_node * --wsnode_tail (struct wordsplit_node *p) --{ -- while (p && p->next) -- p = p->next; -- return p; --} -- - static void - wsnode_insert (struct wordsplit *wsp, struct wordsplit_node *node, - struct wordsplit_node *anchor, int before) -@@ -501,24 +356,22 @@ wsnode_insert (struct wordsplit *wsp, struct wordsplit_node *node, - wsnode_insert (wsp, node, anchor->prev, 0); - else - { -- struct wordsplit_node *tail = wsnode_tail (node); - node->prev = NULL; -- tail->next = anchor; -- anchor->prev = tail; -+ node->next = anchor; -+ anchor->prev = node; - wsp->ws_head = node; - } - } - else - { - struct wordsplit_node *p; -- struct wordsplit_node *tail = wsnode_tail (node); - - p = anchor->next; - if (p) -- p->prev = tail; -+ p->prev = node; - else -- wsp->ws_tail = tail; -- tail->next = p; -+ wsp->ws_tail = node; -+ node->next = p; - node->prev = anchor; - anchor->next = node; - } -@@ -565,12 +418,10 @@ wordsplit_dump_nodes (struct wordsplit *wsp) - for (p = wsp->ws_head, n = 0; p; p = p->next, n++) - { - if (p->flags & _WSNF_WORD) -- wsp->ws_debug ("(%02d) %4d: %p: %#04x (%s):%s;", -- wsp->ws_lvl, -+ wsp->ws_debug ("%4d: %p: %#04x (%s):%s;", - n, p, p->flags, wsnode_flagstr (p->flags), p->v.word); - else -- wsp->ws_debug ("(%02d) %4d: %p: %#04x (%s):%.*s;", -- wsp->ws_lvl, -+ wsp->ws_debug ("%4d: %p: %#04x (%s):%.*s;", - n, p, p->flags, wsnode_flagstr (p->flags), - (int) (p->v.segm.end - p->v.segm.beg), - wsp->ws_input + p->v.segm.beg); -@@ -585,15 +436,11 @@ coalesce_segment (struct wordsplit *wsp, struct wordsplit_node *node) - char *buf, *cur; - int stop; - -- if (!(node->flags & _WSNF_JOIN)) -- return 0; -- - for (p = node; p && (p->flags & _WSNF_JOIN); p = p->next) - { - len += wsnode_len (p); - } -- if (p) -- len += wsnode_len (p); -+ len += wsnode_len (p); - end = p; - - buf = malloc (len + 1); -@@ -612,7 +459,6 @@ coalesce_segment (struct wordsplit *wsp, struct wordsplit_node *node) - cur += slen; - if (p != node) - { -- node->flags |= p->flags & _WSNF_QUOTE; - wsnode_remove (wsp, p); - stop = p == end; - wsnode_free (p); -@@ -632,14 +478,13 @@ coalesce_segment (struct wordsplit *wsp, struct wordsplit_node *node) - return 0; - } - --static void wordsplit_string_unquote_copy (struct wordsplit *ws, int inquote, -- char *dst, const char *src, -- size_t n); -- - static int - wsnode_quoteremoval (struct wordsplit *wsp) - { - struct wordsplit_node *p; -+ void (*uqfn) (char *, const char *, size_t) = -+ (wsp->ws_flags & WRDSF_CESCAPES) ? -+ wordsplit_c_unquote_copy : wordsplit_sh_unquote_copy; - - for (p = wsp->ws_head; p; p = p->next) - { -@@ -648,7 +493,9 @@ wsnode_quoteremoval (struct wordsplit *wsp) - int unquote; - - if (wsp->ws_flags & WRDSF_QUOTE) -- unquote = !(p->flags & _WSNF_NOEXPAND); -+ { -+ unquote = !(p->flags & _WSNF_NOEXPAND); -+ } - else - unquote = 0; - -@@ -665,8 +512,11 @@ wsnode_quoteremoval (struct wordsplit *wsp) - p->flags |= _WSNF_WORD; - } - -- wordsplit_string_unquote_copy (wsp, p->flags & _WSNF_QUOTE, -- p->v.word, str, slen); -+ if (wsp->ws_flags & WRDSF_ESCAPE) -+ wordsplit_general_unquote_copy (p->v.word, str, slen, -+ wsp->ws_escape); -+ else -+ uqfn (p->v.word, str, slen); - } - } - return 0; -@@ -686,161 +536,24 @@ wsnode_coalesce (struct wordsplit *wsp) - return 0; - } - --static int --wsnode_tail_coalesce (struct wordsplit *wsp, struct wordsplit_node *p) --{ -- if (p->next) -- { -- struct wordsplit_node *np = p; -- while (np && np->next) -- { -- np->flags |= _WSNF_JOIN; -- np = np->next; -- } -- if (coalesce_segment (wsp, p)) -- return 1; -- } -- return 0; --} -- --static size_t skip_delim (struct wordsplit *wsp); -- - static int - wordsplit_finish (struct wordsplit *wsp) - { - struct wordsplit_node *p; - size_t n; -- int delim; -- -- /* Postprocess delimiters. It would be rather simple, if it weren't for -- the incremental operation. -- -- Nodes of type _WSNF_DELIM get inserted to the node list if either -- WRDSF_RETURN_DELIMS flag or WRDSO_MAXWORDS option is set. -- -- The following cases should be distinguished: -- -- 1. If both WRDSF_SQUEEZE_DELIMS and WRDSF_RETURN_DELIMS are set, compress -- any runs of similar delimiter nodes to a single node. The nodes are -- 'similar' if they point to the same delimiter character. -- -- If WRDSO_MAXWORDS option is set, stop compressing when -- ws_wordi + 1 == ws_maxwords, and coalesce the rest of nodes into -- a single last node. -- -- 2. If WRDSO_MAXWORDS option is set, but WRDSF_RETURN_DELIMS is not, -- remove any delimiter nodes. Stop operation when -- ws_wordi + 1 == ws_maxwords, and coalesce the rest of nodes into -- a single last node. -- -- 3. If incremental operation is in progress, restart the loop any time -- a delimiter node is about to be returned, unless WRDSF_RETURN_DELIMS -- is set. -- */ -- again: -- delim = 0; /* Delimiter being processed (if any) */ -- n = 0; /* Number of words processed so far */ -- p = wsp->ws_head; /* Current node */ -- -- while (p) -- { -- struct wordsplit_node *next = p->next; -- if (p->flags & _WSNF_DELIM) -- { -- if (wsp->ws_flags & WRDSF_RETURN_DELIMS) -- { -- if (wsp->ws_flags & WRDSF_SQUEEZE_DELIMS) -- { -- char const *s = wsnode_ptr (wsp, p); -- if (delim) -- { -- if (delim == *s) -- { -- wsnode_remove (wsp, p); -- p = next; -- continue; -- } -- else -- { -- delim = 0; -- n++; /* Count this node; it will be returned */ -- } -- } -- else -- { -- delim = *s; -- p = next; -- continue; -- } -- } -- } -- else if (wsp->ws_options & WRDSO_MAXWORDS) -- { -- wsnode_remove (wsp, p); -- p = next; -- continue; -- } -- } -- else -- { -- if (delim) -- { -- /* Last node was a delimiter or a compressed run of delimiters; -- Count it, and clear the delimiter marker */ -- n++; -- delim = 0; -- } -- if (wsp->ws_options & WRDSO_MAXWORDS) -- { -- if (wsp->ws_wordi + n + 1 == wsp->ws_maxwords) -- break; -- } -- } -- n++; -- if (wsp->ws_flags & WRDSF_INCREMENTAL) -- p = NULL; /* Break the loop */ -- else -- p = next; -- } - -- if (p) -- { -- /* We're here if WRDSO_MAXWORDS is in effect and wsp->ws_maxwords -- words have already been collected. Reconstruct a single final -- node from the remaining nodes. */ -- if (wsnode_tail_coalesce (wsp, p)) -- return wsp->ws_errno; -- n++; -- } -+ n = 0; - -- if (n == 0 && (wsp->ws_flags & WRDSF_INCREMENTAL)) -- { -- /* The loop above have eliminated all nodes. Restart the -- processing, if there's any input left. */ -- if (wsp->ws_endp < wsp->ws_len) -- { -- int rc; -- if (wsp->ws_flags & WRDSF_SHOWDBG) -- wsp->ws_debug (_("Restarting")); -- rc = wordsplit_process_list (wsp, skip_delim (wsp)); -- if (rc) -- return rc; -- } -- else -- { -- wsp->ws_error = WRDSE_EOF; -- return WRDSE_EOF; -- } -- goto again; -- } -+ for (p = wsp->ws_head; p; p = p->next) -+ n++; - - if (alloc_space (wsp, n + 1)) -- return wsp->ws_errno; -+ return 1; - -- while (wsp->ws_head) -+ for (p = wsp->ws_head; p; p = p->next) - { -- const char *str = wsnode_ptr (wsp, wsp->ws_head); -- size_t slen = wsnode_len (wsp->ws_head); -+ const char *str = wsnode_ptr (wsp, p); -+ size_t slen = wsnode_len (p); - char *newstr = malloc (slen + 1); - - /* Assign newstr first, even if it is NULL. This way -@@ -852,47 +565,14 @@ wordsplit_finish (struct wordsplit *wsp) - memcpy (newstr, str, slen); - newstr[slen] = 0; - -- wsnode_remove (wsp, wsp->ws_head); -- - wsp->ws_wordc++; -- wsp->ws_wordi++; - -- if (wsp->ws_flags & WRDSF_INCREMENTAL) -- break; - } - wsp->ws_wordv[wsp->ws_offs + wsp->ws_wordc] = NULL; - return 0; - } - --int --wordsplit_append (wordsplit_t *wsp, int argc, char **argv) --{ -- int rc; -- size_t i; - -- rc = alloc_space (wsp, wsp->ws_wordc + argc + 1); -- if (rc) -- return rc; -- for (i = 0; i < argc; i++) -- { -- char *newstr = strdup (argv[i]); -- if (!newstr) -- { -- while (i > 0) -- { -- free (wsp->ws_wordv[wsp->ws_offs + wsp->ws_wordc + i - 1]); -- wsp->ws_wordv[wsp->ws_offs + wsp->ws_wordc + i - 1] = NULL; -- i--; -- } -- return _wsplt_nomem (wsp); -- } -- wsp->ws_wordv[wsp->ws_offs + wsp->ws_wordc + i] = newstr; -- } -- wsp->ws_wordc += i; -- wsp->ws_wordv[wsp->ws_offs + wsp->ws_wordc] = NULL; -- return 0; --} -- - /* Variable expansion */ - static int - node_split_prefix (struct wordsplit *wsp, -@@ -929,10 +609,10 @@ node_split_prefix (struct wordsplit *wsp, - } - - static int --find_closing_paren (const char *str, size_t i, size_t len, size_t *poff, -- char const *paren) -+find_closing_cbrace (const char *str, size_t i, size_t len, size_t * poff) - { -- enum { st_init, st_squote, st_dquote } state = st_init; -+ enum -+ { st_init, st_squote, st_dquote } state = st_init; - size_t level = 1; - - for (; i < len; i++) -@@ -942,23 +622,18 @@ find_closing_paren (const char *str, size_t i, size_t len, size_t *poff, - case st_init: - switch (str[i]) - { -- default: -- if (str[i] == paren[0]) -- { -- level++; -- break; -- } -- else if (str[i] == paren[1]) -+ case '{': -+ level++; -+ break; -+ -+ case '}': -+ if (--level == 0) - { -- if (--level == 0) -- { -- *poff = i; -- return 0; -- } -- break; -+ *poff = i; -+ return 0; - } - break; -- -+ - case '"': - state = st_dquote; - break; -@@ -985,14 +660,13 @@ find_closing_paren (const char *str, size_t i, size_t len, size_t *poff, - return 1; - } - --static int --wordsplit_find_env (struct wordsplit *wsp, const char *name, size_t len, -- char const **ret) -+static const char * -+wordsplit_find_env (struct wordsplit *wsp, const char *name, size_t len) - { - size_t i; - - if (!(wsp->ws_flags & WRDSF_ENV)) -- return WRDSE_UNDEF; -+ return NULL; - - if (wsp->ws_flags & WRDSF_ENV_KV) - { -@@ -1001,17 +675,14 @@ wordsplit_find_env (struct wordsplit *wsp, const char *name, size_t len, - { - size_t elen = strlen (wsp->ws_env[i]); - if (elen == len && memcmp (wsp->ws_env[i], name, elen) == 0) -- { -- *ret = wsp->ws_env[i + 1]; -- return WRDSE_OK; -- } -+ return wsp->ws_env[i + 1]; - /* Skip the value. Break the loop if it is NULL. */ - i++; - if (wsp->ws_env[i] == NULL) - break; - } - } -- else if (wsp->ws_env) -+ else - { - /* Usual (A=B) environment. */ - for (i = 0; wsp->ws_env[i]; i++) -@@ -1023,137 +694,27 @@ wordsplit_find_env (struct wordsplit *wsp, const char *name, size_t len, - if (name[j] != var[j]) - break; - if (j == len && var[j] == '=') -- { -- *ret = var + j + 1; -- return WRDSE_OK; -- } -- } -- } -- return WRDSE_UNDEF; --} -- --static int --wsplt_assign_var (struct wordsplit *wsp, const char *name, size_t namelen, -- char *value) --{ -- int n = (wsp->ws_flags & WRDSF_ENV_KV) ? 2 : 1; -- char *v; -- -- if (wsp->ws_envidx + n >= wsp->ws_envsiz) -- { -- size_t sz; -- char **newenv; -- -- if (!wsp->ws_envbuf) -- { -- if (wsp->ws_flags & WRDSF_ENV) -- { -- size_t i = 0, j; -- -- if (wsp->ws_env) -- { -- for (; wsp->ws_env[i]; i++) -- ; -- } -- -- sz = i + n + 1; -- -- newenv = calloc (sz, sizeof(newenv[0])); -- if (!newenv) -- return _wsplt_nomem (wsp); -- -- for (j = 0; j < i; j++) -- { -- newenv[j] = strdup (wsp->ws_env[j]); -- if (!newenv[j]) -- { -- for (; j > 1; j--) -- free (newenv[j-1]); -- free (newenv[j-1]); -- free (newenv); -- return _wsplt_nomem (wsp); -- } -- } -- newenv[j] = NULL; -- -- wsp->ws_envbuf = newenv; -- wsp->ws_envidx = i; -- wsp->ws_envsiz = sz; -- wsp->ws_env = (const char**) wsp->ws_envbuf; -- } -- else -- { -- newenv = calloc (WORDSPLIT_ENV_INIT, sizeof(newenv[0])); -- if (!newenv) -- return _wsplt_nomem (wsp); -- wsp->ws_envbuf = newenv; -- wsp->ws_envidx = 0; -- wsp->ws_envsiz = WORDSPLIT_ENV_INIT; -- wsp->ws_env = (const char**) wsp->ws_envbuf; -- wsp->ws_flags |= WRDSF_ENV; -- } -- } -- else -- { -- wsp->ws_envsiz *= 2; -- newenv = realloc (wsp->ws_envbuf, -- wsp->ws_envsiz * sizeof (wsp->ws_envbuf[0])); -- if (!newenv) -- return _wsplt_nomem (wsp); -- wsp->ws_envbuf = newenv; -- wsp->ws_env = (const char**) wsp->ws_envbuf; -- } -- } -- -- if (wsp->ws_flags & WRDSF_ENV_KV) -- { -- /* A key-value pair environment */ -- char *p = malloc (namelen + 1); -- if (!p) -- return _wsplt_nomem (wsp); -- memcpy (p, name, namelen); -- p[namelen] = 0; -- -- v = strdup (value); -- if (!v) -- { -- free (p); -- return _wsplt_nomem (wsp); -+ return var + j + 1; - } -- wsp->ws_env[wsp->ws_envidx++] = p; -- wsp->ws_env[wsp->ws_envidx++] = v; -- } -- else -- { -- v = malloc (namelen + strlen(value) + 2); -- if (!v) -- return _wsplt_nomem (wsp); -- memcpy (v, name, namelen); -- v[namelen++] = '='; -- strcpy(v + namelen, value); -- wsp->ws_env[wsp->ws_envidx++] = v; - } -- wsp->ws_env[wsp->ws_envidx++] = NULL; -- return WRDSE_OK; -+ return NULL; - } - - static int - expvar (struct wordsplit *wsp, const char *str, size_t len, -- struct wordsplit_node **ptail, const char **pend, unsigned flg) -+ struct wordsplit_node **ptail, const char **pend, int flg) - { - size_t i = 0; - const char *defstr = NULL; -- char *value; -+ const char *value; - const char *vptr; - struct wordsplit_node *newnode; - const char *start = str - 1; -- int rc; -- struct wordsplit ws; -- -- if (ISVARBEG (str[0])) -+ -+ if (ISALPHA (str[0]) || str[0] == '_') - { - for (i = 1; i < len; i++) -- if (!ISVARCHR (str[i])) -+ if (!(ISALNUM (str[i]) || str[i] == '_')) - break; - *pend = str + i - 1; - } -@@ -1162,36 +723,30 @@ expvar (struct wordsplit *wsp, const char *str, size_t len, - str++; - len--; - for (i = 1; i < len; i++) -+ if (str[i] == '}' || str[i] == ':') -+ break; -+ if (str[i] == ':') - { -- if (str[i] == ':') -- { -- size_t j; -- -- defstr = str + i + 1; -- if (find_closing_paren (str, i + 1, len, &j, "{}")) -- return _wsplt_seterr (wsp, WRDSE_CBRACE); -- *pend = str + j; -- break; -- } -- else if (str[i] == '}') -- { -- defstr = NULL; -- *pend = str + i; -- break; -- } -- else if (strchr ("-+?=", str[i])) -+ size_t j; -+ -+ defstr = str + i + 1; -+ if (find_closing_cbrace (str, i + 1, len, &j)) - { -- size_t j; -- -- defstr = str + i; -- if (find_closing_paren (str, i, len, &j, "{}")) -- return _wsplt_seterr (wsp, WRDSE_CBRACE); -- *pend = str + j; -- break; -+ wsp->ws_errno = WRDSE_CBRACE; -+ return 1; - } -+ *pend = str + j; -+ } -+ else if (str[i] == '}') -+ { -+ defstr = NULL; -+ *pend = str + i; -+ } -+ else -+ { -+ wsp->ws_errno = WRDSE_CBRACE; -+ return 1; - } -- if (i == len) -- return _wsplt_seterr (wsp, WRDSE_CBRACE); - } - else - { -@@ -1215,161 +770,50 @@ expvar (struct wordsplit *wsp, const char *str, size_t len, - i - its length - defstr - default replacement str */ - -- if (defstr && strchr("-+?=", defstr[0]) == 0) -+ vptr = wordsplit_find_env (wsp, str, i); -+ if (vptr) -+ { -+ value = strdup (vptr); -+ if (!value) -+ return _wsplt_nomem (wsp); -+ } -+ else if (wsp->ws_flags & WRDSF_GETVAR) -+ value = wsp->ws_getvar (str, i, wsp->ws_closure); -+ else if (wsp->ws_flags & WRDSF_UNDEF) - { -- rc = WRDSE_UNDEF; -- defstr = NULL; -+ wsp->ws_errno = WRDSE_UNDEF; -+ if (wsp->ws_flags & WRDSF_SHOWERR) -+ wordsplit_perror (wsp); -+ return 1; - } - else - { -- rc = wordsplit_find_env (wsp, str, i, &vptr); -- if (rc == WRDSE_OK) -- { -- if (vptr) -- { -- value = strdup (vptr); -- if (!value) -- rc = WRDSE_NOSPACE; -- } -- else -- rc = WRDSE_UNDEF; -- } -- else if (wsp->ws_flags & WRDSF_GETVAR) -- rc = wsp->ws_getvar (&value, str, i, wsp->ws_closure); -+ if (wsp->ws_flags & WRDSF_WARNUNDEF) -+ wsp->ws_error (_("warning: undefined variable `%.*s'"), (int) i, str); -+ if (wsp->ws_flags & WRDSF_KEEPUNDEF) -+ value = NULL; - else -- rc = WRDSE_UNDEF; -- -- if (rc == WRDSE_OK -- && (!value || value[0] == 0) -- && defstr && defstr[-1] == ':') -- { -- free (value); -- rc = WRDSE_UNDEF; -- } -+ value = ""; - } -- -- switch (rc) -+ -+ /* FIXME: handle defstr */ -+ (void) defstr; -+ -+ if (value) - { -- case WRDSE_OK: -- if (defstr && *defstr == '+') -+ if (flg & _WSNF_QUOTE) - { -- size_t size = *pend - ++defstr; -- -- rc = _wsplt_subsplit (wsp, &ws, defstr, size, -- WRDSF_NOSPLIT | WRDSF_WS | WRDSF_QUOTE | -- (wsp->ws_flags & -- (WRDSF_NOVAR | WRDSF_NOCMD)), 1); -- if (rc) -- return rc; -- free (value); -- value = ws.ws_wordv[0]; -- ws.ws_wordv[0] = NULL; -- wordsplit_free (&ws); -+ if (wsnode_new (wsp, &newnode)) -+ return 1; -+ wsnode_insert (wsp, newnode, *ptail, 0); -+ *ptail = newnode; -+ newnode->flags = _WSNF_WORD | _WSNF_NOEXPAND | flg; -+ newnode->v.word = strdup (value); -+ if (!newnode->v.word) -+ return _wsplt_nomem (wsp); - } -- break; -- -- case WRDSE_UNDEF: -- if (defstr) -+ else if (*value == 0) - { -- size_t size; -- if (*defstr == '-' || *defstr == '=') -- { -- size = *pend - ++defstr; -- -- rc = _wsplt_subsplit (wsp, &ws, defstr, size, -- WRDSF_NOSPLIT | WRDSF_WS | WRDSF_QUOTE | -- (wsp->ws_flags & -- (WRDSF_NOVAR | WRDSF_NOCMD)), -- 1); -- if (rc) -- return rc; -- -- value = ws.ws_wordv[0]; -- ws.ws_wordv[0] = NULL; -- wordsplit_free (&ws); -- -- if (defstr[-1] == '=') -- wsplt_assign_var (wsp, str, i, value); -- } -- else -- { -- if (*defstr == '?') -- { -- size = *pend - ++defstr; -- if (size == 0) -- wsp->ws_error (_("%.*s: variable null or not set"), -- (int) i, str); -- else -- { -- rc = _wsplt_subsplit (wsp, &ws, defstr, size, -- WRDSF_NOSPLIT | WRDSF_WS | -- WRDSF_QUOTE | -- (wsp->ws_flags & -- (WRDSF_NOVAR | WRDSF_NOCMD)), -- 1); -- if (rc == 0) -- wsp->ws_error ("%.*s: %s", -- (int) i, str, ws.ws_wordv[0]); -- else -- wsp->ws_error ("%.*s: %.*s", -- (int) i, str, (int) size, defstr); -- wordsplit_free (&ws); -- } -- } -- value = NULL; -- } -- } -- else if (wsp->ws_flags & WRDSF_UNDEF) -- { -- _wsplt_seterr (wsp, WRDSE_UNDEF); -- return 1; -- } -- else -- { -- if (wsp->ws_flags & WRDSF_WARNUNDEF) -- wsp->ws_error (_("warning: undefined variable `%.*s'"), -- (int) i, str); -- if (wsp->ws_flags & WRDSF_KEEPUNDEF) -- value = NULL; -- else -- { -- value = strdup (""); -- if (!value) -- return _wsplt_nomem (wsp); -- } -- } -- break; -- -- case WRDSE_NOSPACE: -- return _wsplt_nomem (wsp); -- -- case WRDSE_USERERR: -- if (wsp->ws_errno == WRDSE_USERERR) -- free (wsp->ws_usererr); -- wsp->ws_usererr = value; -- FALLTHROUGH; -- default: -- _wsplt_seterr (wsp, rc); -- return 1; -- } -- -- if (value) -- { -- if (flg & _WSNF_QUOTE) -- { -- if (wsnode_new (wsp, &newnode)) -- { -- free (value); -- return 1; -- } -- wsnode_insert (wsp, newnode, *ptail, 0); -- *ptail = newnode; -- newnode->flags = _WSNF_WORD | _WSNF_NOEXPAND | flg; -- newnode->v.word = value; -- } -- else if (*value == 0) -- { -- free (value); - /* Empty string is a special case */ - if (wsnode_new (wsp, &newnode)) - return 1; -@@ -1380,23 +824,28 @@ expvar (struct wordsplit *wsp, const char *str, size_t len, - else - { - struct wordsplit ws; -- int rc; -- -- rc = _wsplt_subsplit (wsp, &ws, value, strlen (value), -- WRDSF_NOVAR | WRDSF_NOCMD | -- WRDSF_QUOTE -- | (WSP_RETURN_DELIMS (wsp) ? WRDSF_RETURN_DELIMS : 0) , -- 0); -- free (value); -- if (rc) -+ int i; -+ -+ ws.ws_delim = wsp->ws_delim; -+ if (wordsplit (value, &ws, -+ WRDSF_NOVAR | WRDSF_NOCMD | WRDSF_DELIM | WRDSF_WS)) - { -- _wsplt_seterr_sub (wsp, &ws); - wordsplit_free (&ws); - return 1; - } -- wsnode_insert (wsp, ws.ws_head, *ptail, 0); -- *ptail = ws.ws_tail; -- ws.ws_head = ws.ws_tail = NULL; -+ for (i = 0; i < ws.ws_wordc; i++) -+ { -+ if (wsnode_new (wsp, &newnode)) -+ return 1; -+ wsnode_insert (wsp, newnode, *ptail, 0); -+ *ptail = newnode; -+ newnode->flags = _WSNF_WORD | -+ _WSNF_NOEXPAND | -+ (i + 1 < ws.ws_wordc ? (flg & ~_WSNF_JOIN) : flg); -+ newnode->v.word = strdup (ws.ws_wordv[i]); -+ if (!newnode->v.word) -+ return _wsplt_nomem (wsp); -+ } - wordsplit_free (&ws); - } - } -@@ -1427,19 +876,7 @@ expvar (struct wordsplit *wsp, const char *str, size_t len, - } - - static int --begin_var_p (int c) --{ -- return c == '{' || ISVARBEG (c); --} -- --static int --node_expand (struct wordsplit *wsp, struct wordsplit_node *node, -- int (*beg_p) (int), -- int (*ws_exp_fn) (struct wordsplit *wsp, -- const char *str, size_t len, -- struct wordsplit_node **ptail, -- const char **pend, -- unsigned flg)) -+node_expand_vars (struct wordsplit *wsp, struct wordsplit_node *node) - { - const char *str = wsnode_ptr (wsp, node); - size_t slen = wsnode_len (node); -@@ -1455,7 +892,7 @@ node_expand (struct wordsplit *wsp, struct wordsplit_node *node, - p++; - continue; - } -- if (*p == '$' && beg_p (p[1])) -+ if (*p == '$') - { - size_t n = p - str; - -@@ -1464,8 +901,8 @@ node_expand (struct wordsplit *wsp, struct wordsplit_node *node, - if (node_split_prefix (wsp, &tail, node, off, n, _WSNF_JOIN)) - return 1; - p++; -- if (ws_exp_fn (wsp, p, slen - n, &tail, &p, -- node->flags & (_WSNF_JOIN | _WSNF_QUOTE))) -+ if (expvar (wsp, p, slen - n, &tail, &p, -+ node->flags & (_WSNF_JOIN | _WSNF_QUOTE))) - return 1; - off += p - str + 1; - str = p + 1; -@@ -1476,7 +913,7 @@ node_expand (struct wordsplit *wsp, struct wordsplit_node *node, - if (tail != node) - tail->flags |= _WSNF_JOIN; - if (node_split_prefix (wsp, &tail, node, off, p - str, -- node->flags & (_WSNF_JOIN|_WSNF_QUOTE))) -+ node->flags & _WSNF_JOIN)) - return 1; - } - if (tail != node) -@@ -1486,8 +923,8 @@ node_expand (struct wordsplit *wsp, struct wordsplit_node *node, - } - return 0; - } -- --/* Remove NULL nodes from the list */ -+ -+/* Remove NULL lists */ - static void - wsnode_nullelim (struct wordsplit *wsp) - { -@@ -1496,8 +933,6 @@ wsnode_nullelim (struct wordsplit *wsp) - for (p = wsp->ws_head; p;) - { - struct wordsplit_node *next = p->next; -- if (p->flags & _WSNF_DELIM && p->prev) -- p->prev->flags &= ~_WSNF_JOIN; - if (p->flags & _WSNF_NULL) - { - wsnode_remove (wsp, p); -@@ -1512,140 +947,11 @@ wordsplit_varexp (struct wordsplit *wsp) - { - struct wordsplit_node *p; - -- for (p = wsp->ws_head; p;) -- { -- struct wordsplit_node *next = p->next; -- if (!(p->flags & (_WSNF_NOEXPAND|_WSNF_DELIM))) -- if (node_expand (wsp, p, begin_var_p, expvar)) -- return 1; -- p = next; -- } -- -- wsnode_nullelim (wsp); -- return 0; --} -- --static int --begin_cmd_p (int c) --{ -- return c == '('; --} -- --static int --expcmd (struct wordsplit *wsp, const char *str, size_t len, -- struct wordsplit_node **ptail, const char **pend, unsigned flg) --{ -- int rc; -- size_t j; -- char *value; -- struct wordsplit_node *newnode; -- -- str++; -- len--; -- -- if (find_closing_paren (str, 0, len, &j, "()")) -- { -- _wsplt_seterr (wsp, WRDSE_PAREN); -- return 1; -- } -- -- *pend = str + j; -- if (wsp->ws_options & WRDSO_ARGV) -- { -- struct wordsplit ws; -- -- rc = _wsplt_subsplit (wsp, &ws, str, j, WRDSF_WS | WRDSF_QUOTE, 1); -- if (rc) -- { -- _wsplt_seterr_sub (wsp, &ws); -- wordsplit_free (&ws); -- return 1; -- } -- rc = wsp->ws_command (&value, str, j, ws.ws_wordv, wsp->ws_closure); -- wordsplit_free (&ws); -- } -- else -- rc = wsp->ws_command (&value, str, j, NULL, wsp->ws_closure); -- -- if (rc == WRDSE_NOSPACE) -- return _wsplt_nomem (wsp); -- else if (rc) -- { -- if (rc == WRDSE_USERERR) -- { -- if (wsp->ws_errno == WRDSE_USERERR) -- free (wsp->ws_usererr); -- wsp->ws_usererr = value; -- } -- _wsplt_seterr (wsp, rc); -- return 1; -- } -- -- if (value) -- { -- if (flg & _WSNF_QUOTE) -- { -- if (wsnode_new (wsp, &newnode)) -- return 1; -- wsnode_insert (wsp, newnode, *ptail, 0); -- *ptail = newnode; -- newnode->flags = _WSNF_WORD | _WSNF_NOEXPAND | flg; -- newnode->v.word = value; -- } -- else if (*value == 0) -- { -- free (value); -- /* Empty string is a special case */ -- if (wsnode_new (wsp, &newnode)) -- return 1; -- wsnode_insert (wsp, newnode, *ptail, 0); -- *ptail = newnode; -- newnode->flags = _WSNF_NULL; -- } -- else -- { -- struct wordsplit ws; -- int rc; -- -- rc = _wsplt_subsplit (wsp, &ws, value, strlen (value), -- WRDSF_NOVAR | WRDSF_NOCMD -- | WRDSF_WS | WRDSF_QUOTE -- | (WSP_RETURN_DELIMS (wsp) ? WRDSF_RETURN_DELIMS : 0), -- 0); -- free (value); -- if (rc) -- { -- _wsplt_seterr_sub (wsp, &ws); -- wordsplit_free (&ws); -- return 1; -- } -- wsnode_insert (wsp, ws.ws_head, *ptail, 0); -- *ptail = ws.ws_tail; -- ws.ws_head = ws.ws_tail = NULL; -- wordsplit_free (&ws); -- } -- } -- else -- { -- if (wsnode_new (wsp, &newnode)) -- return 1; -- wsnode_insert (wsp, newnode, *ptail, 0); -- *ptail = newnode; -- newnode->flags = _WSNF_NULL; -- } -- return 0; --} -- --static int --wordsplit_cmdexp (struct wordsplit *wsp) --{ -- struct wordsplit_node *p; -- - for (p = wsp->ws_head; p;) - { - struct wordsplit_node *next = p->next; - if (!(p->flags & _WSNF_NOEXPAND)) -- if (node_expand (wsp, p, begin_cmd_p, expcmd)) -+ if (node_expand_vars (wsp, p)) - return 1; - p = next; - } -@@ -1657,7 +963,7 @@ wordsplit_cmdexp (struct wordsplit *wsp) - /* Strip off any leading and trailing whitespace. This function is called - right after the initial scanning, therefore it assumes that every - node in the list is a text reference node. */ --static int -+static void - wordsplit_trimws (struct wordsplit *wsp) - { - struct wordsplit_node *p; -@@ -1666,21 +972,14 @@ wordsplit_trimws (struct wordsplit *wsp) - { - size_t n; - -- if (!(p->flags & _WSNF_QUOTE)) -- { -- /* Skip leading whitespace: */ -- for (n = p->v.segm.beg; n < p->v.segm.end && ISWS (wsp->ws_input[n]); -- n++) -- ; -- p->v.segm.beg = n; -- } -- -- while (p->next && (p->flags & _WSNF_JOIN)) -- p = p->next; -- - if (p->flags & _WSNF_QUOTE) - continue; -- -+ -+ /* Skip leading whitespace: */ -+ for (n = p->v.segm.beg; n < p->v.segm.end && ISWS (wsp->ws_input[n]); -+ n++) -+ ; -+ p->v.segm.beg = n; - /* Trim trailing whitespace */ - for (n = p->v.segm.end; - n > p->v.segm.beg && ISWS (wsp->ws_input[n - 1]); n--); -@@ -1690,194 +989,6 @@ wordsplit_trimws (struct wordsplit *wsp) - } - - wsnode_nullelim (wsp); -- return 0; --} -- --static int --wordsplit_tildexpand (struct wordsplit *wsp) --{ -- struct wordsplit_node *p; -- char *uname = NULL; -- size_t usize = 0; -- -- for (p = wsp->ws_head; p; p = p->next) -- { -- const char *str; -- -- if (p->flags & _WSNF_QUOTE) -- continue; -- -- str = wsnode_ptr (wsp, p); -- if (str[0] == '~') -- { -- size_t i, size, dlen; -- size_t slen = wsnode_len (p); -- struct passwd *pw; -- char *newstr; -- -- for (i = 1; i < slen && str[i] != '/'; i++) -- ; -- if (i == slen) -- continue; -- if (i > 1) -- { -- if (i > usize) -- { -- char *p = realloc (uname, i); -- if (!p) -- { -- free (uname); -- return _wsplt_nomem (wsp); -- } -- uname = p; -- usize = i; -- } -- --i; -- memcpy (uname, str + 1, i); -- uname[i] = 0; -- pw = getpwnam (uname); -- } -- else -- pw = getpwuid (getuid ()); -- -- if (!pw) -- continue; -- -- dlen = strlen (pw->pw_dir); -- size = slen - i + dlen; -- newstr = malloc (size); -- if (!newstr) -- { -- free (uname); -- return _wsplt_nomem (wsp); -- } -- --size; -- -- memcpy (newstr, pw->pw_dir, dlen); -- memcpy (newstr + dlen, str + i + 1, slen - i - 1); -- newstr[size] = 0; -- if (p->flags & _WSNF_WORD) -- free (p->v.word); -- p->v.word = newstr; -- p->flags |= _WSNF_WORD; -- } -- } -- free (uname); -- return 0; --} -- --static int --isglob (const char *s, int l) --{ -- while (l--) -- { -- if (strchr ("*?[", *s++)) -- return 1; -- } -- return 0; --} -- --static int --wordsplit_pathexpand (struct wordsplit *wsp) --{ -- struct wordsplit_node *p, *next; -- size_t slen; -- int flags = 0; -- --#ifdef GLOB_PERIOD -- if (wsp->ws_options & WRDSO_DOTGLOB) -- flags = GLOB_PERIOD; --#endif -- -- for (p = wsp->ws_head; p; p = next) -- { -- const char *str; -- -- next = p->next; -- -- if (p->flags & _WSNF_QUOTE) -- continue; -- -- str = wsnode_ptr (wsp, p); -- slen = wsnode_len (p); -- -- if (isglob (str, slen)) -- { -- int i; -- glob_t g; -- struct wordsplit_node *prev; -- char *pattern; -- -- pattern = malloc (slen + 1); -- if (!pattern) -- return _wsplt_nomem (wsp); -- memcpy (pattern, str, slen); -- pattern[slen] = 0; -- -- switch (glob (pattern, flags, NULL, &g)) -- { -- case 0: -- free (pattern); -- break; -- -- case GLOB_NOSPACE: -- free (pattern); -- return _wsplt_nomem (wsp); -- -- case GLOB_NOMATCH: -- if (wsp->ws_options & WRDSO_NULLGLOB) -- { -- wsnode_remove (wsp, p); -- wsnode_free (p); -- } -- else if (wsp->ws_options & WRDSO_FAILGLOB) -- { -- char buf[128]; -- if (wsp->ws_errno == WRDSE_USERERR) -- free (wsp->ws_usererr); -- snprintf (buf, sizeof (buf), _("no files match pattern %s"), -- pattern); -- free (pattern); -- wsp->ws_usererr = strdup (buf); -- if (!wsp->ws_usererr) -- return _wsplt_nomem (wsp); -- else -- return _wsplt_seterr (wsp, WRDSE_USERERR); -- } -- free (pattern); -- continue; -- -- default: -- free (pattern); -- return _wsplt_seterr (wsp, WRDSE_GLOBERR); -- } -- -- prev = p; -- for (i = 0; i < g.gl_pathc; i++) -- { -- struct wordsplit_node *newnode; -- char *newstr; -- -- if (wsnode_new (wsp, &newnode)) -- return 1; -- newstr = strdup (g.gl_pathv[i]); -- if (!newstr) -- { -- wsnode_free (newnode); -- return _wsplt_nomem (wsp); -- } -- newnode->v.word = newstr; -- newnode->flags |= _WSNF_WORD|_WSNF_QUOTE; -- wsnode_insert (wsp, newnode, prev, 0); -- prev = newnode; -- } -- globfree (&g); -- -- wsnode_remove (wsp, p); -- wsnode_free (p); -- } -- } -- return 0; - } - - static int -@@ -1913,24 +1024,33 @@ skip_sed_expr (const char *command, size_t i, size_t len) - return i; - } - --/* wsp->ws_endp points to a delimiter character. If RETURN_DELIMS -- is true, return its value, otherwise return the index past it. */ --static inline size_t --skip_delim_internal (struct wordsplit *wsp, int return_delims) --{ -- return return_delims ? wsp->ws_endp : wsp->ws_endp + 1; --} -- --static inline size_t -+static size_t - skip_delim (struct wordsplit *wsp) - { -- return skip_delim_internal (wsp, WSP_RETURN_DELIMS (wsp)); --} -+ size_t start = wsp->ws_endp; -+ if (wsp->ws_flags & WRDSF_SQUEEZE_DELIMS) -+ { -+ if ((wsp->ws_flags & WRDSF_RETURN_DELIMS) && -+ ISDELIM (wsp, wsp->ws_input[start])) -+ { -+ int delim = wsp->ws_input[start]; -+ do -+ start++; -+ while (start < wsp->ws_len && delim == wsp->ws_input[start]); -+ } -+ else -+ { -+ do -+ start++; -+ while (start < wsp->ws_len && ISDELIM (wsp, wsp->ws_input[start])); -+ } -+ start--; -+ } - --static inline size_t --skip_delim_real (struct wordsplit *wsp) --{ -- return skip_delim_internal (wsp, wsp->ws_flags & WRDSF_RETURN_DELIMS); -+ if (!(wsp->ws_flags & WRDSF_RETURN_DELIMS)) -+ start++; -+ -+ return start; - } - - #define _WRDS_EOF 0 -@@ -1938,7 +1058,7 @@ skip_delim_real (struct wordsplit *wsp) - #define _WRDS_ERR 2 - - static int --scan_qstring (struct wordsplit *wsp, size_t start, size_t *end) -+scan_qstring (struct wordsplit *wsp, size_t start, size_t * end) - { - size_t j; - const char *command = wsp->ws_input; -@@ -1950,7 +1070,7 @@ scan_qstring (struct wordsplit *wsp, size_t start, size_t *end) - j++; - if (j < len && command[j] == q) - { -- unsigned flags = _WSNF_QUOTE | _WSNF_EMPTYOK; -+ int flags = _WSNF_QUOTE | _WSNF_EMPTYOK; - if (q == '\'') - flags |= _WSNF_NOEXPAND; - if (wordsplit_add_segm (wsp, start + 1, j, flags)) -@@ -1960,22 +1080,23 @@ scan_qstring (struct wordsplit *wsp, size_t start, size_t *end) - else - { - wsp->ws_endp = start; -- _wsplt_seterr (wsp, WRDSE_QUOTE); -+ wsp->ws_errno = WRDSE_QUOTE; -+ if (wsp->ws_flags & WRDSF_SHOWERR) -+ wordsplit_perror (wsp); - return _WRDS_ERR; - } - return 0; - } - - static int --scan_word (struct wordsplit *wsp, size_t start, int consume_all) -+scan_word (struct wordsplit *wsp, size_t start) - { - size_t len = wsp->ws_len; - const char *command = wsp->ws_input; - const char *comment = wsp->ws_comment; - int join = 0; -- unsigned flags = 0; -- struct wordsplit_node *np = wsp->ws_tail; -- -+ int flags = 0; -+ - size_t i = start; - - if (i >= len) -@@ -1992,7 +1113,7 @@ scan_word (struct wordsplit *wsp, size_t start, int consume_all) - flags = _WSNF_SEXP; - i = skip_sed_expr (command, i, len); - } -- else if (consume_all || !ISDELIM (wsp, command[i])) -+ else if (!ISDELIM (wsp, command[i])) - { - while (i < len) - { -@@ -2031,28 +1152,15 @@ scan_word (struct wordsplit *wsp, size_t start, int consume_all) - } - } - -- if (command[i] == '$') -- { -- if (!(wsp->ws_flags & WRDSF_NOVAR) -- && command[i+1] == '{' -- && find_closing_paren (command, i + 2, len, &i, "{}") == 0) -- continue; -- if (!(wsp->ws_flags & WRDSF_NOCMD) -- && command[i+1] == '(' -- && find_closing_paren (command, i + 2, len, &i, "()") == 0) -- continue; -- } -- -- if (!consume_all && ISDELIM (wsp, command[i])) -+ if (ISDELIM (wsp, command[i])) - break; - else - i++; - } - } -- else if (WSP_RETURN_DELIMS (wsp)) -+ else if (wsp->ws_flags & WRDSF_RETURN_DELIMS) - { - i++; -- flags |= _WSNF_DELIM; - } - else if (!(wsp->ws_flags & WRDSF_SQUEEZE_DELIMS)) - flags |= _WSNF_EMPTYOK; -@@ -2064,19 +1172,36 @@ scan_word (struct wordsplit *wsp, size_t start, int consume_all) - wsp->ws_endp = i; - if (wsp->ws_flags & WRDSF_INCREMENTAL) - return _WRDS_EOF; -- -- if (consume_all) -+ return _WRDS_OK; -+} -+ -+static char quote_transtab[] = "\\\\\"\"a\ab\bf\fn\nr\rt\tv\v"; -+ -+int -+wordsplit_c_unquote_char (int c) -+{ -+ char *p; -+ -+ for (p = quote_transtab; *p; p += 2) - { -- if (!np) -- np = wsp->ws_head; -- while (np) -- { -- np->flags |= _WSNF_QUOTE; -- np = np->next; -- } -+ if (*p == c) -+ return p[1]; - } -- -- return _WRDS_OK; -+ return c; -+} -+ -+int -+wordsplit_c_quote_char (int c) -+{ -+ char *p; -+ -+ for (p = quote_transtab + sizeof (quote_transtab) - 2; -+ p > quote_transtab; p -= 2) -+ { -+ if (*p == c) -+ return p[-1]; -+ } -+ return -1; - } - - #define to_num(c) \ -@@ -2119,7 +1244,7 @@ wordsplit_c_quoted_length (const char *str, int quote_hex, int *quote) - len += 3; - else - { -- if (wordsplit_c_quote_char (*str)) -+ if (wordsplit_c_quote_char (*str) != -1) - len += 2; - else - len += 4; -@@ -2128,56 +1253,47 @@ wordsplit_c_quoted_length (const char *str, int quote_hex, int *quote) - return len; - } - --static int --wsplt_unquote_char (const char *transtab, int c) -+void -+wordsplit_general_unquote_copy (char *dst, const char *src, size_t n, -+ const char *escapable) - { -- while (*transtab && transtab[1]) -- { -- if (*transtab++ == c) -- return *transtab; -- ++transtab; -- } -- return 0; --} -+ int i; - --static int --wsplt_quote_char (const char *transtab, int c) --{ -- for (; *transtab && transtab[1]; transtab += 2) -+ for (i = 0; i < n;) - { -- if (transtab[1] == c) -- return *transtab; -+ if (src[i] == '\\' && i < n && strchr (escapable, src[i + 1])) -+ i++; -+ *dst++ = src[i++]; - } -- return 0; -+ *dst = 0; - } - --int --wordsplit_c_unquote_char (int c) -+void -+wordsplit_sh_unquote_copy (char *dst, const char *src, size_t n) - { -- return wsplt_unquote_char (wordsplit_c_escape_tab, c); --} -+ int i; - --int --wordsplit_c_quote_char (int c) --{ -- return wsplt_quote_char (wordsplit_c_escape_tab, c); -+ for (i = 0; i < n;) -+ { -+ if (src[i] == '\\') -+ i++; -+ *dst++ = src[i++]; -+ } -+ *dst = 0; - } - - void --wordsplit_string_unquote_copy (struct wordsplit *ws, int inquote, -- char *dst, const char *src, size_t n) -+wordsplit_c_unquote_copy (char *dst, const char *src, size_t n) - { - int i = 0; - int c; - -- inquote = !!inquote; - while (i < n) - { - if (src[i] == '\\') - { - ++i; -- if (WRDSO_ESC_TEST (ws, inquote, WRDSO_XESC) -- && (src[i] == 'x' || src[i] == 'X')) -+ if (src[i] == 'x' || src[i] == 'X') - { - if (n - i < 2) - { -@@ -2200,8 +1316,7 @@ wordsplit_string_unquote_copy (struct wordsplit *ws, int inquote, - } - } - } -- else if (WRDSO_ESC_TEST (ws, inquote, WRDSO_OESC) -- && (unsigned char) src[i] < 128 && ISDIGIT (src[i])) -+ else if ((unsigned char) src[i] < 128 && ISDIGIT (src[i])) - { - if (n - i < 1) - { -@@ -2223,17 +1338,8 @@ wordsplit_string_unquote_copy (struct wordsplit *ws, int inquote, - } - } - } -- else if ((c = wsplt_unquote_char (ws->ws_escape[inquote], src[i]))) -- { -- *dst++ = c; -- ++i; -- } - else -- { -- if (WRDSO_ESC_TEST (ws, inquote, WRDSO_BSKEEP)) -- *dst++ = '\\'; -- *dst++ = src[i++]; -- } -+ *dst++ = wordsplit_c_unquote_char (src[i++]); - } - else - *dst++ = src[i++]; -@@ -2267,7 +1373,7 @@ wordsplit_c_quote_copy (char *dst, const char *src, int quote_hex) - { - int c = wordsplit_c_quote_char (*src); - *dst++ = '\\'; -- if (c) -+ if (c != -1) - *dst++ = c; - else - { -@@ -2280,82 +1386,20 @@ wordsplit_c_quote_copy (char *dst, const char *src, int quote_hex) - } - } - -- --/* This structure describes a single expansion phase */ --struct exptab --{ -- char const *descr; /* Textual description (for debugging) */ -- int flag; /* WRDSF_ bit that controls this phase */ -- int opt; /* Entry-specific options (see EXPOPT_ flags below */ -- int (*expansion) (struct wordsplit *wsp); /* expansion function */ --}; -- --/* The following options control expansions: */ --/* Normally the exptab entry is run if its flag bit is set in struct -- wordsplit. The EXPOPT_NEG option negates this test so that expansion -- is performed if its associated flag bit is not set in struct wordsplit. */ --#define EXPOPT_NEG 0x01 --/* All bits in flag must be set in order for entry to match */ --#define EXPORT_ALLOF 0x02 --/* Coalesce the input list before running the expansion. */ --#define EXPOPT_COALESCE 0x04 -- --static struct exptab exptab[] = { -- { N_("WS trimming"), WRDSF_WS, 0, -- wordsplit_trimws }, -- { N_("command substitution"), WRDSF_NOCMD, EXPOPT_NEG|EXPOPT_COALESCE, -- wordsplit_cmdexp }, -- { N_("coalesce list"), 0, EXPOPT_NEG|EXPOPT_COALESCE, -- NULL }, -- { N_("tilde expansion"), WRDSF_PATHEXPAND, 0, -- wordsplit_tildexpand }, -- { N_("variable expansion"), WRDSF_NOVAR, EXPOPT_NEG, -- wordsplit_varexp }, -- { N_("quote removal"), 0, EXPOPT_NEG, -- wsnode_quoteremoval }, -- { N_("coalesce list"), 0, EXPOPT_NEG|EXPOPT_COALESCE, -- NULL }, -- { N_("path expansion"), WRDSF_PATHEXPAND, 0, -- wordsplit_pathexpand }, -- { NULL } --}; -- --static inline int --exptab_matches(struct exptab *p, struct wordsplit *wsp) --{ -- int result; -- -- result = (wsp->ws_flags & p->flag); -- if (p->opt & EXPORT_ALLOF) -- result = result == p->flag; -- if (p->opt & EXPOPT_NEG) -- result = !result; -- -- return result; --} -- - static int - wordsplit_process_list (struct wordsplit *wsp, size_t start) - { -- struct exptab *p; -- -- if (wsp->ws_flags & WRDSF_SHOWDBG) -- wsp->ws_debug (_("(%02d) Input:%.*s;"), -- wsp->ws_lvl, (int) wsp->ws_len, wsp->ws_input); -- -- if ((wsp->ws_flags & WRDSF_NOSPLIT) -- || ((wsp->ws_options & WRDSO_MAXWORDS) -- && wsp->ws_wordi + 1 == wsp->ws_maxwords)) -- { -- /* Treat entire input as a single word */ -- if (scan_word (wsp, start, 1) == _WRDS_ERR) -+ if (wsp->ws_flags & WRDSF_NOSPLIT) -+ { -+ /* Treat entire input as a quoted argument */ -+ if (wordsplit_add_segm (wsp, start, wsp->ws_len, _WSNF_QUOTE)) - return wsp->ws_errno; - } - else - { - int rc; - -- while ((rc = scan_word (wsp, start, 0)) == _WRDS_OK) -+ while ((rc = scan_word (wsp, start)) == _WRDS_OK) - start = skip_delim (wsp); - /* Make sure tail element is not joinable */ - if (wsp->ws_tail) -@@ -2366,88 +1410,132 @@ wordsplit_process_list (struct wordsplit *wsp, size_t start) - - if (wsp->ws_flags & WRDSF_SHOWDBG) - { -- wsp->ws_debug ("(%02d) %s", wsp->ws_lvl, _("Initial list:")); -+ wsp->ws_debug ("Initial list:"); - wordsplit_dump_nodes (wsp); - } - -- for (p = exptab; p->descr; p++) -+ if (wsp->ws_flags & WRDSF_WS) - { -- if (exptab_matches(p, wsp)) -+ /* Trim leading and trailing whitespace */ -+ wordsplit_trimws (wsp); -+ if (wsp->ws_flags & WRDSF_SHOWDBG) - { -- if (p->opt & EXPOPT_COALESCE) -- { -- if (wsnode_coalesce (wsp)) -- break; -- if (wsp->ws_flags & WRDSF_SHOWDBG) -- { -- wsp->ws_debug ("(%02d) %s", wsp->ws_lvl, -- _("Coalesced list:")); -- wordsplit_dump_nodes (wsp); -- } -- } -- if (p->expansion) -- { -- if (p->expansion (wsp)) -- break; -- if (wsp->ws_flags & WRDSF_SHOWDBG) -- { -- wsp->ws_debug ("(%02d) %s", wsp->ws_lvl, _(p->descr)); -- wordsplit_dump_nodes (wsp); -- } -- } -+ wsp->ws_debug ("After WS trimming:"); -+ wordsplit_dump_nodes (wsp); -+ } -+ } -+ -+ /* Expand variables (FIXME: & commands) */ -+ if (!(wsp->ws_flags & WRDSF_NOVAR)) -+ { -+ if (wordsplit_varexp (wsp)) -+ { -+ wordsplit_free_nodes (wsp); -+ return wsp->ws_errno; -+ } -+ if (wsp->ws_flags & WRDSF_SHOWDBG) -+ { -+ wsp->ws_debug ("Expanded list:"); -+ wordsplit_dump_nodes (wsp); -+ } -+ } -+ -+ do -+ { -+ if (wsnode_quoteremoval (wsp)) -+ break; -+ if (wsp->ws_flags & WRDSF_SHOWDBG) -+ { -+ wsp->ws_debug ("After quote removal:"); -+ wordsplit_dump_nodes (wsp); -+ } -+ -+ if (wsnode_coalesce (wsp)) -+ break; -+ -+ if (wsp->ws_flags & WRDSF_SHOWDBG) -+ { -+ wsp->ws_debug ("Coalesced list:"); -+ wordsplit_dump_nodes (wsp); - } - } -+ while (0); - return wsp->ws_errno; - } - --static int --wordsplit_run (const char *command, size_t length, struct wordsplit *wsp, -- unsigned flags, int lvl) -+int -+wordsplit_len (const char *command, size_t length, struct wordsplit *wsp, -+ int flags) - { - int rc; - size_t start; -+ const char *cmdptr; -+ size_t cmdlen; - - if (!command) - { - if (!(flags & WRDSF_INCREMENTAL)) -- return _wsplt_seterr (wsp, WRDSE_USAGE); -- -- if (wsp->ws_head) -- return wordsplit_finish (wsp); -+ return EINVAL; - -- start = skip_delim_real (wsp); -+ start = skip_delim (wsp); - if (wsp->ws_endp == wsp->ws_len) -- return _wsplt_seterr (wsp, WRDSE_NOINPUT); -+ { -+ wsp->ws_errno = WRDSE_NOINPUT; -+ if (wsp->ws_flags & WRDSF_SHOWERR) -+ wordsplit_perror (wsp); -+ return wsp->ws_errno; -+ } - -+ cmdptr = wsp->ws_input + wsp->ws_endp; -+ cmdlen = wsp->ws_len - wsp->ws_endp; - wsp->ws_flags |= WRDSF_REUSE; - wordsplit_init0 (wsp); - } - else - { -+ cmdptr = command; -+ cmdlen = length; - start = 0; -- rc = wordsplit_init (wsp, command, length, flags); -+ rc = wordsplit_init (wsp, cmdptr, cmdlen, flags); - if (rc) - return rc; -- wsp->ws_lvl = lvl; - } - -+ if (wsp->ws_flags & WRDSF_SHOWDBG) -+ wsp->ws_debug ("Input:%.*s;", (int) cmdlen, cmdptr); -+ - rc = wordsplit_process_list (wsp, start); -+ if (rc == 0 && (flags & WRDSF_INCREMENTAL)) -+ { -+ while (!wsp->ws_head && wsp->ws_endp < wsp->ws_len) -+ { -+ start = skip_delim (wsp); -+ if (wsp->ws_flags & WRDSF_SHOWDBG) -+ { -+ cmdptr = wsp->ws_input + wsp->ws_endp; -+ cmdlen = wsp->ws_len - wsp->ws_endp; -+ wsp->ws_debug ("Restart:%.*s;", (int) cmdlen, cmdptr); -+ } -+ rc = wordsplit_process_list (wsp, start); -+ if (rc) -+ break; -+ } -+ } - if (rc) -- return rc; -- return wordsplit_finish (wsp); --} -- --int --wordsplit_len (const char *command, size_t length, struct wordsplit *wsp, -- unsigned flags) --{ -- return wordsplit_run (command, length, wsp, flags, 0); -+ { -+ wordsplit_free_nodes (wsp); -+ return rc; -+ } -+ wordsplit_finish (wsp); -+ wordsplit_free_nodes (wsp); -+ return wsp->ws_errno; - } - - int --wordsplit (const char *command, struct wordsplit *ws, unsigned flags) -+wordsplit (const char *command, struct wordsplit *ws, int flags) - { -- return wordsplit_len (command, command ? strlen (command) : 0, ws, flags); -+ return wordsplit_len (command, command ? strlen (command) : 0, ws, -+ flags); - } - - void -@@ -2467,69 +1555,67 @@ wordsplit_free_words (struct wordsplit *ws) - ws->ws_wordc = 0; - } - --void --wordsplit_free_envbuf (struct wordsplit *ws) --{ -- if (ws->ws_flags & WRDSF_NOCMD) -- return; -- if (ws->ws_envbuf) -- { -- size_t i; -- -- for (i = 0; ws->ws_envbuf[i]; i++) -- free (ws->ws_envbuf[i]); -- free (ws->ws_envbuf); -- ws->ws_envidx = ws->ws_envsiz = 0; -- ws->ws_envbuf = NULL; -- } --} -- --void --wordsplit_clearerr (struct wordsplit *ws) --{ -- if (ws->ws_errno == WRDSE_USERERR) -- free (ws->ws_usererr); -- ws->ws_usererr = NULL; -- ws->ws_errno = WRDSE_OK; --} -- - void - wordsplit_free (struct wordsplit *ws) - { -- wordsplit_free_nodes (ws); - wordsplit_free_words (ws); - free (ws->ws_wordv); - ws->ws_wordv = NULL; -- wordsplit_free_envbuf (ws); - } - --int --wordsplit_get_words (struct wordsplit *ws, size_t *wordc, char ***wordv) -+void -+wordsplit_perror (struct wordsplit *wsp) - { -- char **p = realloc (ws->ws_wordv, -- (ws->ws_wordc + 1) * sizeof (ws->ws_wordv[0])); -- if (!p) -- return -1; -- *wordv = p; -- *wordc = ws->ws_wordc; -+ switch (wsp->ws_errno) -+ { -+ case WRDSE_EOF: -+ wsp->ws_error (_("no error")); -+ break; - -- ws->ws_wordv = NULL; -- ws->ws_wordc = 0; -- ws->ws_wordn = 0; -+ case WRDSE_QUOTE: -+ wsp->ws_error (_("missing closing %c (start near #%lu)"), -+ wsp->ws_input[wsp->ws_endp], -+ (unsigned long) wsp->ws_endp); -+ break; - -- return 0; -+ case WRDSE_NOSPACE: -+ wsp->ws_error (_("memory exhausted")); -+ break; -+ -+ case WRDSE_NOSUPP: -+ wsp->ws_error (_("command substitution is not yet supported")); -+ break; -+ -+ case WRDSE_USAGE: -+ wsp->ws_error (_("invalid wordsplit usage")); -+ break; -+ -+ case WRDSE_CBRACE: -+ wsp->ws_error (_("unbalanced curly brace")); -+ break; -+ -+ case WRDSE_UNDEF: -+ wsp->ws_error (_("undefined variable")); -+ break; -+ -+ case WRDSE_NOINPUT: -+ wsp->ws_error (_("input exhausted")); -+ break; -+ -+ default: -+ wsp->ws_error (_("unknown error")); -+ } - } - - const char *_wordsplit_errstr[] = { - N_("no error"), - N_("missing closing quote"), - N_("memory exhausted"), -+ N_("command substitution is not yet supported"), - N_("invalid wordsplit usage"), - N_("unbalanced curly brace"), - N_("undefined variable"), -- N_("input exhausted"), -- N_("unbalanced parenthesis"), -- N_("globbing error") -+ N_("input exhausted") - }; - int _wordsplit_nerrs = - sizeof (_wordsplit_errstr) / sizeof (_wordsplit_errstr[0]); -@@ -2537,26 +1623,7 @@ int _wordsplit_nerrs = - const char * - wordsplit_strerror (struct wordsplit *ws) - { -- if (ws->ws_errno == WRDSE_USERERR) -- return ws->ws_usererr; - if (ws->ws_errno < _wordsplit_nerrs) - return _wordsplit_errstr[ws->ws_errno]; - return N_("unknown error"); - } -- --void --wordsplit_perror (struct wordsplit *wsp) --{ -- switch (wsp->ws_errno) -- { -- case WRDSE_QUOTE: -- wsp->ws_error (_("missing closing %c (start near #%lu)"), -- wsp->ws_input[wsp->ws_endp], -- (unsigned long) wsp->ws_endp); -- break; -- -- default: -- wsp->ws_error ("%s", wordsplit_strerror (wsp)); -- } --} -- -diff --git a/lib/wordsplit.h b/lib/wordsplit.h -index ff7f6e3..575865b 100644 ---- a/lib/wordsplit.h -+++ b/lib/wordsplit.h -@@ -1,5 +1,5 @@ - /* wordsplit - a word splitter -- Copyright (C) 2009-2018 Sergey Poznyakoff -+ Copyright (C) 2009-2014, 2016-2017 Free Software Foundation, Inc. - - This program is free software; you can redistribute it and/or modify it - under the terms of the GNU General Public License as published by the -@@ -12,7 +12,10 @@ - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along -- with this program. If not, see . */ -+ with this program. If not, see . -+ -+ Written by Sergey Poznyakoff -+*/ - - #ifndef __WORDSPLIT_H - #define __WORDSPLIT_H -@@ -25,106 +28,42 @@ - # define __WORDSPLIT_ATTRIBUTE_FORMAT(spec) /* empty */ - #endif - --typedef struct wordsplit wordsplit_t; -- --/* Structure used to direct the splitting. Members marked with [Input] -- can be defined before calling wordsplit(), those marked with [Output] -- provide return values when the function returns. If neither mark is -- used, the member is internal and must not be used by the caller. -- -- In the comments below, the identifiers in parentheses indicate bits that -- must be set (or unset, if starting with !) in ws_flags (if starting with -- WRDSF_) or ws_options (if starting with WRDSO_) to initialize or use the -- given member. -- -- If not redefined explicitly, most of them are set to some reasonable -- default value upon entry to wordsplit(). */ --struct wordsplit -+struct wordsplit - { -- size_t ws_wordc; /* [Output] Number of words in ws_wordv. */ -- char **ws_wordv; /* [Output] Array of parsed out words. */ -- size_t ws_offs; /* [Input] (WRDSF_DOOFFS) Number of initial -- elements in ws_wordv to fill with NULLs. */ -- size_t ws_wordn; /* Number of elements ws_wordv can accomodate. */ -- unsigned ws_flags; /* [Input] Flags passed to wordsplit. */ -- unsigned ws_options; /* [Input] (WRDSF_OPTIONS) -- Additional options. */ -- size_t ws_maxwords; /* [Input] (WRDSO_MAXWORDS) Return at most that -- many words */ -- size_t ws_wordi; /* [Output] (WRDSF_INCREMENTAL) Total number of -- words returned so far */ -- -- const char *ws_delim; /* [Input] (WRDSF_DELIM) Word delimiters. */ -- const char *ws_comment; /* [Input] (WRDSF_COMMENT) Comment characters. */ -- const char *ws_escape[2]; /* [Input] (WRDSF_ESCAPE) Characters to be escaped -- with backslash. */ -- void (*ws_alloc_die) (wordsplit_t *wsp); -- /* [Input] (WRDSF_ALLOC_DIE) Function called when -- out of memory. Must not return. */ -+ size_t ws_wordc; -+ char **ws_wordv; -+ size_t ws_offs; -+ size_t ws_wordn; -+ int ws_flags; -+ const char *ws_delim; -+ const char *ws_comment; -+ const char *ws_escape; -+ void (*ws_alloc_die) (struct wordsplit * wsp); - void (*ws_error) (const char *, ...) -- __attribute__ ((__format__ (__printf__, 1, 2))); -- /* [Input] (WRDSF_ERROR) Function used for error -- reporting */ -+ __WORDSPLIT_ATTRIBUTE_FORMAT ((__printf__, 1, 2)); - void (*ws_debug) (const char *, ...) -- __attribute__ ((__format__ (__printf__, 1, 2))); -- /* [Input] (WRDSF_DEBUG) Function used for debug -- output. */ -- const char **ws_env; /* [Input] (WRDSF_ENV, !WRDSF_NOVAR) Array of -- environment variables. */ -+ __WORDSPLIT_ATTRIBUTE_FORMAT ((__printf__, 1, 2)); - -- char **ws_envbuf; -- size_t ws_envidx; -- size_t ws_envsiz; -- -- int (*ws_getvar) (char **ret, const char *var, size_t len, void *clos); -- /* [Input] (WRDSF_GETVAR, !WRDSF_NOVAR) Looks up -- the name VAR (LEN bytes long) in the table of -- variables and if found returns in memory -- location pointed to by RET the value of that -- variable. Returns WRDSE_OK (0) on success, -- and an error code (see WRDSE_* defines below) -- on error. User-specific errors can be returned -- by storing the error diagnostic string in RET -- and returning WRDSE_USERERR. -- Whatever is stored in RET, it must be allocated -- using malloc(3). */ -- void *ws_closure; /* [Input] (WRDSF_CLOSURE) Passed as the CLOS -- argument to ws_getvar and ws_command. */ -- int (*ws_command) (char **ret, const char *cmd, size_t len, char **argv, -- void *clos); -- /* [Input] (!WRDSF_NOCMD) Returns in the memory -- location pointed to by RET the expansion of -- the command CMD (LEN bytes long). If WRDSO_ARGV -- option is set, ARGV contains CMD split out to -- words. Otherwise ARGV is NULL. -+ const char **ws_env; -+ const char *(*ws_getvar) (const char *, size_t, void *); -+ void *ws_closure; - -- See ws_getvar for a discussion of possible -- return values. */ -- -- const char *ws_input; /* Input string (the S argument to wordsplit. */ -- size_t ws_len; /* Length of ws_input. */ -- size_t ws_endp; /* Points past the last processed byte in -- ws_input. */ -- int ws_errno; /* [Output] Error code, if an error occurred. */ -- char *ws_usererr; /* Points to textual description of -- the error, if ws_errno is WRDSE_USERERR. Must -- be allocated with malloc(3). */ -+ const char *ws_input; -+ size_t ws_len; -+ size_t ws_endp; -+ int ws_errno; - struct wordsplit_node *ws_head, *ws_tail; -- /* Doubly-linked list of parsed out nodes. */ -- int ws_lvl; /* Invocation nesting level. */ - }; - --/* Initial size for ws_env, if allocated automatically */ --#define WORDSPLIT_ENV_INIT 16 -- --/* Wordsplit flags. */ -+/* Wordsplit flags. Only 2 bits of a 32-bit word remain unused. -+ It is getting crowded... */ - /* Append the words found to the array resulting from a previous - call. */ - #define WRDSF_APPEND 0x00000001 --/* Insert ws_offs initial NULLs in the array ws_wordv. -+/* Insert we_offs initial NULLs in the array ws_wordv. - (These are not counted in the returned ws_wordc.) */ - #define WRDSF_DOOFFS 0x00000002 --/* Don't do command substitution. */ -+/* Don't do command substitution. Reserved for future use. */ - #define WRDSF_NOCMD 0x00000004 - /* The parameter p resulted from a previous call to - wordsplit(), and wordsplit_free() was not called. Reuse the -@@ -132,8 +71,10 @@ struct wordsplit - #define WRDSF_REUSE 0x00000008 - /* Print errors */ - #define WRDSF_SHOWERR 0x00000010 --/* Consider it an error if an undefined variable is expanded. */ -+/* Consider it an error if an undefined shell variable -+ is expanded. */ - #define WRDSF_UNDEF 0x00000020 -+ - /* Don't do variable expansion. */ - #define WRDSF_NOVAR 0x00000040 - /* Abort on ENOMEM error */ -@@ -144,7 +85,7 @@ struct wordsplit - #define WRDSF_SQUOTE 0x00000200 - /* Handle double quotes */ - #define WRDSF_DQUOTE 0x00000400 --/* Handle single and double quotes */ -+/* Handle quotes and escape directives */ - #define WRDSF_QUOTE (WRDSF_SQUOTE|WRDSF_DQUOTE) - /* Replace each input sequence of repeated delimiters with a single - delimiter */ -@@ -172,106 +113,56 @@ struct wordsplit - /* Don't split input into words. Useful for side effects. */ - #define WRDSF_NOSPLIT 0x00400000 - /* Keep undefined variables in place, instead of expanding them to -- empty strings. */ -+ empty string */ - #define WRDSF_KEEPUNDEF 0x00800000 - /* Warn about undefined variables */ - #define WRDSF_WARNUNDEF 0x01000000 - /* Handle C escapes */ - #define WRDSF_CESCAPES 0x02000000 -+ - /* ws_closure is set */ - #define WRDSF_CLOSURE 0x04000000 - /* ws_env is a Key/Value environment, i.e. the value of a variable is - stored in the element that follows its name. */ - #define WRDSF_ENV_KV 0x08000000 -+ - /* ws_escape is set */ - #define WRDSF_ESCAPE 0x10000000 -+ - /* Incremental mode */ - #define WRDSF_INCREMENTAL 0x20000000 --/* Perform pathname and tilde expansion */ --#define WRDSF_PATHEXPAND 0x40000000 --/* ws_options is initialized */ --#define WRDSF_OPTIONS 0x80000000 - - #define WRDSF_DEFFLAGS \ - (WRDSF_NOVAR | WRDSF_NOCMD | \ - WRDSF_QUOTE | WRDSF_SQUEEZE_DELIMS | WRDSF_CESCAPES) - --/* Remove the word that produces empty string after path expansion */ --#define WRDSO_NULLGLOB 0x00000001 --/* Print error message if path expansion produces empty string */ --#define WRDSO_FAILGLOB 0x00000002 --/* Allow a leading period to be matched by metacharacters. */ --#define WRDSO_DOTGLOB 0x00000004 --/* ws_command needs argv parameter */ --#define WRDSO_ARGV 0x00000008 --/* Keep backslash in unrecognized escape sequences in words */ --#define WRDSO_BSKEEP_WORD 0x00000010 --/* Handle octal escapes in words */ --#define WRDSO_OESC_WORD 0x00000020 --/* Handle hex escapes in words */ --#define WRDSO_XESC_WORD 0x00000040 -- --/* ws_maxwords field is initialized */ --#define WRDSO_MAXWORDS 0x00000080 -- --/* Keep backslash in unrecognized escape sequences in quoted strings */ --#define WRDSO_BSKEEP_QUOTE 0x00000100 --/* Handle octal escapes in quoted strings */ --#define WRDSO_OESC_QUOTE 0x00000200 --/* Handle hex escapes in quoted strings */ --#define WRDSO_XESC_QUOTE 0x00000400 -- --#define WRDSO_BSKEEP WRDSO_BSKEEP_WORD --#define WRDSO_OESC WRDSO_OESC_WORD --#define WRDSO_XESC WRDSO_XESC_WORD -- --/* Indices into ws_escape */ --#define WRDSX_WORD 0 --#define WRDSX_QUOTE 1 -- --/* Set escape option F in WS for words (Q==0) or quoted strings (Q==1) */ --#define WRDSO_ESC_SET(ws,q,f) ((ws)->ws_options |= ((f) << 4*(q))) --/* Test WS for escape option F for words (Q==0) or quoted strings (Q==1) */ --#define WRDSO_ESC_TEST(ws,q,f) ((ws)->ws_options & ((f) << 4*(q))) -- --#define WRDSE_OK 0 --#define WRDSE_EOF WRDSE_OK -+#define WRDSE_EOF 0 - #define WRDSE_QUOTE 1 - #define WRDSE_NOSPACE 2 --#define WRDSE_USAGE 3 --#define WRDSE_CBRACE 4 --#define WRDSE_UNDEF 5 --#define WRDSE_NOINPUT 6 --#define WRDSE_PAREN 7 --#define WRDSE_GLOBERR 8 --#define WRDSE_USERERR 9 -- --int wordsplit (const char *s, wordsplit_t *ws, unsigned flags); --int wordsplit_len (const char *s, size_t len, wordsplit_t *ws, unsigned flags); --void wordsplit_free (wordsplit_t *ws); --void wordsplit_free_words (wordsplit_t *ws); --void wordsplit_free_envbuf (wordsplit_t *ws); --int wordsplit_get_words (wordsplit_t *ws, size_t *wordc, char ***wordv); -- --static inline void wordsplit_getwords (wordsplit_t *ws, size_t *wordc, char ***wordv) -- __attribute__ ((deprecated)); -- --static inline void --wordsplit_getwords (wordsplit_t *ws, size_t *wordc, char ***wordv) --{ -- wordsplit_get_words (ws, wordc, wordv); --} -- --int wordsplit_append (wordsplit_t *wsp, int argc, char **argv); -+#define WRDSE_NOSUPP 3 -+#define WRDSE_USAGE 4 -+#define WRDSE_CBRACE 5 -+#define WRDSE_UNDEF 6 -+#define WRDSE_NOINPUT 7 -+ -+int wordsplit (const char *s, struct wordsplit *p, int flags); -+int wordsplit_len (const char *s, size_t len, -+ struct wordsplit *p, int flags); -+void wordsplit_free (struct wordsplit *p); -+void wordsplit_free_words (struct wordsplit *ws); - - int wordsplit_c_unquote_char (int c); - int wordsplit_c_quote_char (int c); --size_t wordsplit_c_quoted_length (const char *str, int quote_hex, int *quote); -+size_t wordsplit_c_quoted_length (const char *str, int quote_hex, -+ int *quote); -+void wordsplit_general_unquote_copy (char *dst, const char *src, size_t n, -+ const char *escapable); -+void wordsplit_sh_unquote_copy (char *dst, const char *src, size_t n); -+void wordsplit_c_unquote_copy (char *dst, const char *src, size_t n); - void wordsplit_c_quote_copy (char *dst, const char *src, int quote_hex); - --void wordsplit_perror (wordsplit_t *ws); --const char *wordsplit_strerror (wordsplit_t *ws); -+void wordsplit_perror (struct wordsplit *ws); -+const char *wordsplit_strerror (struct wordsplit *ws); - --void wordsplit_clearerr (wordsplit_t *ws); - - #endif diff --git a/thirdparty/tar/tar-1.33-remove-o_path-usage.patch b/thirdparty/tar/tar-1.33-remove-o_path-usage.patch deleted file mode 100644 index 66f720373..000000000 --- a/thirdparty/tar/tar-1.33-remove-o_path-usage.patch +++ /dev/null @@ -1,24 +0,0 @@ -diff -Nuarp tar-1.33-ori/gnu/fchmodat.c tar-1.33/gnu/fchmodat.c ---- tar-1.33-ori/gnu/fchmodat.c 2021-01-07 15:29:44.000000000 +0100 -+++ tar-1.33/gnu/fchmodat.c 2021-01-16 00:41:39.414012004 +0100 -@@ -68,7 +68,7 @@ fchmodat (int dir, char const *file, mod - { - struct stat st; - --# if defined O_PATH && defined AT_EMPTY_PATH -+# if defined NOPE_PATH && defined AT_EMPTY_PATH - /* Open a file descriptor with O_NOFOLLOW, to make sure we don't - follow symbolic links, if /proc is mounted. O_PATH is used to - avoid a failure if the file is not readable. -diff -Nuarp tar-1.33-ori/gnu/lchmod.c tar-1.33/gnu/lchmod.c ---- tar-1.33-ori/gnu/lchmod.c 2021-01-07 15:28:45.000000000 +0100 -+++ tar-1.33/gnu/lchmod.c 2021-01-16 00:41:28.708012124 +0100 -@@ -45,7 +45,7 @@ - int - lchmod (char const *file, mode_t mode) - { --#if defined O_PATH && defined AT_EMPTY_PATH -+#if defined NOPE_PATH && defined AT_EMPTY_PATH - /* Open a file descriptor with O_NOFOLLOW, to make sure we don't - follow symbolic links, if /proc is mounted. O_PATH is used to - avoid a failure if the file is not readable. From 361ffa71a5f4650b77261a547e45ecdb02f033be Mon Sep 17 00:00:00 2001 From: Benoit Pierre Date: Sun, 31 Aug 2025 22:05:47 +0200 Subject: [PATCH 07/17] zsync2: drop project Not necessary anymore with the new zipsync OTA mechanism. --- cmake/CMakeLists.txt | 8 --- thirdparty/zsync2/CMakeLists.txt | 38 ------------- thirdparty/zsync2/cmake_tweaks.patch | 85 ---------------------------- 3 files changed, 131 deletions(-) delete mode 100644 thirdparty/zsync2/CMakeLists.txt delete mode 100644 thirdparty/zsync2/cmake_tweaks.patch diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 6229956c4..0a2487e4e 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -437,14 +437,6 @@ declare_project(thirdparty/zlib) # zstd declare_project(thirdparty/zstd) -# zsync2 -if(CERVANTES OR KINDLE OR KOBO OR POCKETBOOK OR REMARKABLE OR SONY_PRSTUX) - set(EXCLUDE_FROM_ALL) -else() - set(EXCLUDE_FROM_ALL EXCLUDE_FROM_ALL) -endif() -declare_project(thirdparty/zsync2 DEPENDS curl libressl zlib ${EXCLUDE_FROM_ALL}) - # }}} # TESTSUITE DEPENDENCIES. {{{ diff --git a/thirdparty/zsync2/CMakeLists.txt b/thirdparty/zsync2/CMakeLists.txt deleted file mode 100644 index 8b28a9f72..000000000 --- a/thirdparty/zsync2/CMakeLists.txt +++ /dev/null @@ -1,38 +0,0 @@ -list(APPEND PATCH_FILES cmake_tweaks.patch) - -list(APPEND CMAKE_ARGS - -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} - -DOPENSSL_USE_STATIC_LIBS=$ - # Options. - -DBUILD_CPR_TESTS=FALSE - -DBUILD_TESTING=FALSE - -DCPR_FORCE_OPENSSL_BACKEND=TRUE - -DUSE_SYSTEM_CURL=TRUE -) - -list(APPEND BUILD_CMD COMMAND ninja zsync2) - -append_binary_install_command(INSTALL_CMD src/zsync2) - -# NOTE: We're currently using our own fork instead of upstream's (https://github.com/AppImage/zsync2): -## * Enable range requests optimizations on the CLI -## * Re-implement the -u flag for compatibility w/ the old zsync CLI -## * Minor CLI output cleanup -## * Squish all the warnings -## * Prevent passing a malformed referer URL -## * Don't print seven billion progress bars -## * Rebase against zsync master (c.f., rebase-zsync-libs branch) -## * Rebase against zlib 1.3.0.1 (c.f., rebase-zlib branch) -## * Workaround potential download loops, like what happened w/ the 2019.12 -> 2020.01 update. -## * Mangle logging to play nice w/ FBInk -## * Plug memory leaks -# c.f., https://github.com/NiLuJe/zsync2 for more details. - -external_project( - DOWNLOAD GIT e281e1eb4466ff6b3866c25dbe62a3e150fa5bfd - https://github.com/NiLuJe/zsync2.git - PATCH_FILES ${PATCH_FILES} - CMAKE_ARGS ${CMAKE_ARGS} - BUILD_COMMAND ${BUILD_CMD} - INSTALL_COMMAND ${INSTALL_CMD} -) diff --git a/thirdparty/zsync2/cmake_tweaks.patch b/thirdparty/zsync2/cmake_tweaks.patch deleted file mode 100644 index e6e311c66..000000000 --- a/thirdparty/zsync2/cmake_tweaks.patch +++ /dev/null @@ -1,85 +0,0 @@ ---- i/CMakeLists.txt -+++ w/CMakeLists.txt -@@ -1,4 +1,4 @@ --cmake_minimum_required(VERSION 3.2) -+cmake_minimum_required(VERSION 3.17.5) - - project(zsync2) - ---- i/lib/cpr/CMakeLists.txt -+++ w/lib/cpr/CMakeLists.txt -@@ -147,13 +147,10 @@ if(CPR_ENABLE_SSL) - endif() - else() - if(CPR_FORCE_OPENSSL_BACKEND) -- find_package(OpenSSL) -- if(OPENSSL_FOUND) -- message(STATUS "Using OpenSSL.") -- set(SSL_BACKEND_USED "OpenSSL") -- else() -- message(FATAL_ERROR "CPR_FORCE_OPENSSL_BACKEND enabled but we were not able to find OpenSSL!") -- endif() -+ find_package(PkgConfig REQUIRED) -+ pkg_check_modules(OpenSSL openssl REQUIRED IMPORTED_TARGET) -+ message(STATUS "Using OpenSSL.") -+ set(SSL_BACKEND_USED "OpenSSL") - elseif(CPR_FORCE_WINSSL_BACKEND) - message(STATUS "Using WinSSL.") - set(SSL_BACKEND_USED "WinSSL") -@@ -170,37 +167,13 @@ if(CPR_ENABLE_SSL) - endif() - - if(SSL_BACKEND_USED STREQUAL "OpenSSL") --# Fix missing OpenSSL includes for Windows since in 'ssl_ctx.cpp' we include OpenSSL directly --find_package(OpenSSL REQUIRED) - add_compile_definitions(OPENSSL_BACKEND_USED) - endif() - - # Curl configuration - if(CPR_USE_SYSTEM_CURL) -- if(CPR_ENABLE_SSL) -- find_package(CURL COMPONENTS HTTP HTTPS) -- if(CURL_FOUND) -- message(STATUS "Curl ${CURL_VERSION_STRING} found on this system.") -- # To be able to load certificates under Windows when using OpenSSL: -- if(CMAKE_USE_OPENSSL AND WIN32 AND (NOT (CURL_VERSION_STRING VERSION_GREATER_EQUAL "7.71.0"))) -- message(FATAL_ERROR "Your system curl version (${CURL_VERSION_STRING}) is too old to support OpenSSL on Windows which requires curl >= 7.71.0. Update your curl version, use WinSSL, disable SSL or use the built-in version of curl.") -- endif() -- else() -- find_package(CURL COMPONENTS HTTP) -- if(CURL_FOUND) -- message(FATAL_ERROR "Curl found on this system but WITHOUT HTTPS/SSL support. Either disable SSL by setting CPR_ENABLE_SSL to OFF or use the built-in version of curl by setting CPR_USE_SYSTEM_CURL to OFF.") -- else() -- message(FATAL_ERROR "Curl not found on this system. To use the built-in version set CPR_USE_SYSTEM_CURL to OFF.") -- endif() -- endif() -- else() -- find_package(CURL COMPONENTS HTTP) -- if(CURL_FOUND) -- message(STATUS "Curl found on this system.") -- else() -- message(FATAL_ERROR "Curl not found on this system. To use the built-in version set CPR_USE_SYSTEM_CURL to OFF.") -- endif() -- endif() -+ find_package(PkgConfig REQUIRED) -+ pkg_check_modules(CURL libcurl REQUIRED IMPORTED_TARGET) - else() - message(STATUS "Configuring built-in curl...") - ---- i/lib/cpr/cpr/CMakeLists.txt -+++ w/lib/cpr/cpr/CMakeLists.txt -@@ -32,12 +32,11 @@ add_library(cpr - - add_library(cpr::cpr ALIAS cpr) - --target_link_libraries(cpr PUBLIC CURL::libcurl) # todo should be private, but first dependencies in ssl_options need to be removed -+target_link_libraries(cpr PUBLIC PkgConfig::CURL) # todo should be private, but first dependencies in ssl_options need to be removed - - # Fix missing OpenSSL includes for Windows since in 'ssl_ctx.cpp' we include OpenSSL directly - if(SSL_BACKEND_USED STREQUAL "OpenSSL") -- target_link_libraries(cpr PRIVATE OpenSSL::SSL) -- target_include_directories(cpr PRIVATE ${OPENSSL_INCLUDE_DIR}) -+ target_link_libraries(cpr PRIVATE PkgConfig::OpenSSL) - endif() - - # Set version for shared libraries. From 868587a8969df738531a121770b29c9afb2fdf1c Mon Sep 17 00:00:00 2001 From: Benoit Pierre Date: Sun, 31 Aug 2025 22:05:48 +0200 Subject: [PATCH 08/17] curl: drop project Not used anymore. --- cmake/CMakeLists.txt | 3 - thirdparty/curl/CMakeLists.txt | 92 ------------------- thirdparty/curl/clock_gettime_needs_rt.patch | 14 --- thirdparty/curl/fetch-ca-bundle.sh | 26 ------ .../curl/no_glibc_2.6_or_above_symbols.patch | 11 --- thirdparty/curl/use_-pthread.patch | 25 ----- 6 files changed, 171 deletions(-) delete mode 100644 thirdparty/curl/CMakeLists.txt delete mode 100644 thirdparty/curl/clock_gettime_needs_rt.patch delete mode 100755 thirdparty/curl/fetch-ca-bundle.sh delete mode 100644 thirdparty/curl/no_glibc_2.6_or_above_symbols.patch delete mode 100644 thirdparty/curl/use_-pthread.patch diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 0a2487e4e..5a8bfbb3c 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -197,9 +197,6 @@ set(crengine_BINARY_DIR ${crengine_CMAKE_BINARY_DIR}/build) # cpu_features declare_project(thirdparty/cpu_features EXCLUDE_FROM_ALL) -# curl -declare_project(thirdparty/curl DEPENDS libressl zlib EXCLUDE_FROM_ALL) - # czmq if(NOT WIN32) set(EXCLUDE_FROM_ALL) diff --git a/thirdparty/curl/CMakeLists.txt b/thirdparty/curl/CMakeLists.txt deleted file mode 100644 index 40f9e3514..000000000 --- a/thirdparty/curl/CMakeLists.txt +++ /dev/null @@ -1,92 +0,0 @@ -list(APPEND PATCH_FILES - # Need `-lrt` for `clock_gettime` support. - clock_gettime_needs_rt.patch - # Use `-pthread` flag, not `-lpthread` - # to avoid conflicts with OpenSSL. - use_-pthread.patch -) -if(LEGACY OR POCKETBOOK) - # Avoid pulling-in `eventfd@GLIBC_2.7` or `pipe2@GLIBC_2.9`. - list(APPEND PATCH_FILES no_glibc_2.6_or_above_symbols.patch) -endif() - -list(APPEND CMAKE_ARGS - -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} - # Build a static library, since there's only one user (zsync2). - -DBUILD_SHARED_LIBS=FALSE - # Project options. - # TODO: Enable ZSTD support? We currently only use cURL - # for zsync2, so, obviously, not necessary right now… - -DBUILD_CURL_EXE=FALSE - -DBUILD_LIBCURL_DOCS=FALSE - -DBUILD_TESTING=FALSE - -DCURL_CA_BUNDLE=./data/ca-bundle.crt - -DCURL_CA_FALLBACK=TRUE - -DCURL_CA_PATH=/etc/ssl/certs - -DCURL_DISABLE_ALTSVC=TRUE - -DCURL_DISABLE_AWS=TRUE - -DCURL_DISABLE_COOKIES=TRUE - -DCURL_DISABLE_DICT=TRUE - -DCURL_DISABLE_DOH=TRUE - -DCURL_DISABLE_FILE=TRUE - -DCURL_DISABLE_FORM_API=TRUE - -DCURL_DISABLE_FTP=TRUE - -DCURL_DISABLE_GOPHER=TRUE - -DCURL_DISABLE_HSTS=TRUE - -DCURL_DISABLE_IMAP=TRUE - -DCURL_DISABLE_IPFS=TRUE - -DCURL_DISABLE_LDAP=TRUE - -DCURL_DISABLE_LDAPS=TRUE - -DCURL_DISABLE_MIME=TRUE - -DCURL_DISABLE_MQTT=TRUE - -DCURL_DISABLE_NETRC=TRUE - -DCURL_DISABLE_NTLM=TRUE - -DCURL_DISABLE_POP3=TRUE - -DCURL_DISABLE_PROGRESS_METER=TRUE - -DCURL_DISABLE_RTSP=TRUE - -DCURL_DISABLE_SHUFFLE_DNS=TRUE - -DCURL_DISABLE_SMB=TRUE - -DCURL_DISABLE_SMTP=TRUE - -DCURL_DISABLE_TELNET=TRUE - -DCURL_DISABLE_TFTP=TRUE - -DCURL_DISABLE_VERBOSE_STRINGS=$> - -DCURL_DISABLE_WEBSOCKETS=TRUE - -DCURL_USE_LIBPSL=FALSE - -DCURL_USE_LIBSSH2=FALSE - -DENABLE_CURL_MANUAL=FALSE - -DUSE_LIBIDN2=FALSE - -DUSE_NGHTTP2=FALSE - # Brotli. - -DCURL_BROTLI=FALSE - # OpenSSL. - -DCURL_DEFAULT_SSL_BACKEND=openssl - -DCURL_USE_OPENSSL=TRUE - # ZLIB. - -DCURL_ZLIB=TRUE - # ZSTD. - -DCURL_ZSTD=FALSE -) - -list(APPEND BUILD_CMD COMMAND ninja lib/all) - -list(APPEND INSTALL_CMD COMMAND ${CMAKE_COMMAND} --install .) - -append_install_commands(INSTALL_CMD ${DOWNLOAD_DIR}/ca-bundle.crt DESTINATION data) - -external_project( - DOWNLOAD URL cba9ea54bccefed639a529b1b5b17405 - https://github.com/curl/curl/releases/download/curl-8_14_1/curl-8.14.1.tar.xz - PATCH_FILES ${PATCH_FILES} - CMAKE_ARGS ${CMAKE_ARGS} - BUILD_COMMAND ${BUILD_CMD} - INSTALL_COMMAND ${INSTALL_CMD} -) - -# Don't use the default build system rule to create the certificates -# bundle: connections to `hg.mozilla.org` from CIs seem to be flaky, -# resulting in regular failures. -external_project_step( - download-ca-bundle BEFORE download - COMMENT "Fetching certificates bundles for '${PROJECT_NAME}'" - COMMAND ${CMAKE_CURRENT_LIST_DIR}/fetch-ca-bundle.sh ${DOWNLOAD_DIR}/ca-bundle.crt -) diff --git a/thirdparty/curl/clock_gettime_needs_rt.patch b/thirdparty/curl/clock_gettime_needs_rt.patch deleted file mode 100644 index 26b0058e9..000000000 --- a/thirdparty/curl/clock_gettime_needs_rt.patch +++ /dev/null @@ -1,14 +0,0 @@ ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -1964,7 +1964,11 @@ - curl_internal_test(HAVE_POSIX_STRERROR_R) - - if(NOT WIN32) -+ set(CMAKE_REQUIRED_LIBRARIES -lrt) - curl_internal_test(HAVE_CLOCK_GETTIME_MONOTONIC) # Check clock_gettime(CLOCK_MONOTONIC, x) support -+ if(HAVE_CLOCK_GETTIME_MONOTONIC) -+ list(APPEND CURL_LIBS "rt") -+ endif() - endif() - - if(APPLE) diff --git a/thirdparty/curl/fetch-ca-bundle.sh b/thirdparty/curl/fetch-ca-bundle.sh deleted file mode 100755 index 6264826ea..000000000 --- a/thirdparty/curl/fetch-ca-bundle.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh - -output="$1" -outdir="$(dirname "${output}")" - -mkdir -p "${outdir}" && curl \ - --etag-compare "${output}.etag" \ - --etag-save "${output}.etag" \ - --output "${output}" \ - --retry 3 \ - https://curl.se/ca/cacert.pem -code=$? - -if [ ${code} -ne 0 ]; then - if [ -r "${output}" ]; then - code=0 - severity=WARNING - else - severity=ERROR - fi - printf '\033[31;1m%s:\033[0m failed to fetch “%s”\n' "${severity}" "${output}" 1>&2 -fi - -exit ${code} - -# vim: sw=4 diff --git a/thirdparty/curl/no_glibc_2.6_or_above_symbols.patch b/thirdparty/curl/no_glibc_2.6_or_above_symbols.patch deleted file mode 100644 index 10fb0c6dd..000000000 --- a/thirdparty/curl/no_glibc_2.6_or_above_symbols.patch +++ /dev/null @@ -1,11 +0,0 @@ ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -1796,8 +1796,6 @@ - check_symbol_exists("getifaddrs" "${CURL_INCLUDES};stdlib.h" HAVE_GETIFADDRS) # ifaddrs.h - check_symbol_exists("freeaddrinfo" "${CURL_INCLUDES}" HAVE_FREEADDRINFO) # ws2tcpip.h sys/socket.h netdb.h - check_function_exists("pipe" HAVE_PIPE) --check_function_exists("pipe2" HAVE_PIPE2) --check_function_exists("eventfd" HAVE_EVENTFD) - check_symbol_exists("ftruncate" "unistd.h" HAVE_FTRUNCATE) - check_symbol_exists("getpeername" "${CURL_INCLUDES}" HAVE_GETPEERNAME) # winsock2.h unistd.h proto/bsdsocket.h - check_symbol_exists("getsockname" "${CURL_INCLUDES}" HAVE_GETSOCKNAME) # winsock2.h unistd.h proto/bsdsocket.h diff --git a/thirdparty/curl/use_-pthread.patch b/thirdparty/curl/use_-pthread.patch deleted file mode 100644 index b3b12862a..000000000 --- a/thirdparty/curl/use_-pthread.patch +++ /dev/null @@ -1,25 +0,0 @@ ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -590,6 +590,7 @@ - if(WIN32) - set(USE_THREADS_WIN32 ON) - else() -+ set(THREADS_PREFER_PTHREAD_FLAG TRUE) - find_package(Threads REQUIRED) - set(USE_THREADS_POSIX ${CMAKE_USE_PTHREADS_INIT}) - set(HAVE_PTHREAD_H ${CMAKE_USE_PTHREADS_INIT}) -@@ -2377,12 +2379,12 @@ - if(BUILD_SHARED_LIBS) - set(ENABLE_SHARED "yes") - set(LIBCURL_PC_REQUIRES "") -- set(LIBCURL_PC_LIBS "") -+ set(LIBCURL_PC_LIBS "-pthread") - set(LIBCURL_PC_CFLAGS "") - else() - set(ENABLE_SHARED "no") - set(LIBCURL_PC_REQUIRES "${LIBCURL_PC_REQUIRES_PRIVATE}") -- set(LIBCURL_PC_LIBS "${LIBCURL_PC_LIBS_PRIVATE}") -+ set(LIBCURL_PC_LIBS "-pthread ${LIBCURL_PC_LIBS_PRIVATE}") - set(LIBCURL_PC_CFLAGS "${LIBCURL_PC_CFLAGS_PRIVATE}") - endif() - if(BUILD_STATIC_LIBS) From 6745a6230e857f2e20d118bf1996a9c3794b6355 Mon Sep 17 00:00:00 2001 From: Benoit Pierre Date: Sun, 31 Aug 2025 22:05:49 +0200 Subject: [PATCH 09/17] ffi/posix: minor cleanup --- ffi/posix_h.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ffi/posix_h.lua b/ffi/posix_h.lua index 8922184ab..99263c3ae 100644 --- a/ffi/posix_h.lua +++ b/ffi/posix_h.lua @@ -57,7 +57,7 @@ int mq_open(const char *, int, ...); ssize_t mq_receive(int, char *, size_t, unsigned int *); int mq_close(int); int close(int); -off_t lseek(int fildes, off_t offset, int whence); +off_t lseek(int, off_t, int); int fcntl(int, int, ...); int execl(const char *, const char *, ...); int execlp(const char *, const char *, ...); From a8ac779bbe195abcbffe97d1a694156783d0c0af Mon Sep 17 00:00:00 2001 From: Benoit Pierre Date: Sun, 31 Aug 2025 22:05:50 +0200 Subject: [PATCH 10/17] ffi/posix: add `ftruncate` & `posix_fallocate` --- ffi/posix_h.lua | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ffi/posix_h.lua b/ffi/posix_h.lua index 99263c3ae..0c9caf44f 100644 --- a/ffi/posix_h.lua +++ b/ffi/posix_h.lua @@ -58,6 +58,8 @@ ssize_t mq_receive(int, char *, size_t, unsigned int *); int mq_close(int); int close(int); off_t lseek(int, off_t, int); +int ftruncate(int, off_t); +int posix_fallocate(int, off_t, off_t); int fcntl(int, int, ...); int execl(const char *, const char *, ...); int execlp(const char *, const char *, ...); From 70f35d6f0c0beea677e46ee9b3e8df36a8c5f5b9 Mon Sep 17 00:00:00 2001 From: Benoit Pierre Date: Sun, 31 Aug 2025 22:05:51 +0200 Subject: [PATCH 11/17] ffi/posix: add `mkstemps` --- ffi/posix_h.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/ffi/posix_h.lua b/ffi/posix_h.lua index 0c9caf44f..9cbcd8a34 100644 --- a/ffi/posix_h.lua +++ b/ffi/posix_h.lua @@ -61,6 +61,7 @@ off_t lseek(int, off_t, int); int ftruncate(int, off_t); int posix_fallocate(int, off_t, off_t); int fcntl(int, int, ...); +int mkstemps(char *, int); int execl(const char *, const char *, ...); int execlp(const char *, const char *, ...); int execv(const char *, char *const *); From a8cfb9d41b70582503f187032ad8ad8f6eff35fa Mon Sep 17 00:00:00 2001 From: Benoit Pierre Date: Sun, 31 Aug 2025 22:05:52 +0200 Subject: [PATCH 12/17] ffi/zstd: allow setting compression level --- ffi/zstd.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ffi/zstd.lua b/ffi/zstd.lua index 20b62500a..ed8243b4b 100644 --- a/ffi/zstd.lua +++ b/ffi/zstd.lua @@ -15,13 +15,13 @@ local zst = ffi.loadlib("zstd", "1") local zstd = {} -- c.f., https://github.com/facebook/zstd/tree/dev/examples -function zstd.zstd_compress(ptr, size) +function zstd.zstd_compress(ptr, size, level) --print("zstd_compress:", ptr, size) local n = zst.ZSTD_compressBound(size) local cbuff = C.calloc(n, 1) assert(cbuff ~= nil, "Failed to allocate ZSTD compression buffer (" .. tonumber(n) .. " bytes)") -- NOTE: We should be quite all right with the default (3), which will most likely trounce zlib's 9 in every respect... - local clen = zst.ZSTD_compress(cbuff, n, ptr, size, zst.ZSTD_CLEVEL_DEFAULT) + local clen = zst.ZSTD_compress(cbuff, n, ptr, size, level or zst.ZSTD_CLEVEL_DEFAULT) if zst.ZSTD_isError(clen) ~= 0 then C.free(cbuff) error(ffi.string(zst.ZSTD_getErrorName(clen))) From 3e7f058dbc8c2a52e356308d06aeb2a8817cad67 Mon Sep 17 00:00:00 2001 From: Benoit Pierre Date: Sun, 31 Aug 2025 22:05:54 +0200 Subject: [PATCH 13/17] ffi/xz: add cdecls --- cmake/CMakeLists.txt | 3 + ffi-cdecl/xz_cdecl.c | 76 +++++++++++++++++++ ffi/xz_h.lua | 174 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 253 insertions(+) create mode 100644 ffi-cdecl/xz_cdecl.c create mode 100644 ffi/xz_h.lua diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 5a8bfbb3c..18c455083 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -629,6 +629,9 @@ ffi_target(utf8proc utf8proc_h.lua utf8proc_decl.c ARGS -d libutf8proc) # xxhash ffi_target(xxhash xxhash_h.lua xxhash_cdecl.c) +# xz +ffi_target(xz xz_h.lua xz_cdecl.c) + # zlib ffi_target(zlib zlib_h.lua zlib_decl.c ARGS -d zlib) diff --git a/ffi-cdecl/xz_cdecl.c b/ffi-cdecl/xz_cdecl.c new file mode 100644 index 000000000..65e44adc5 --- /dev/null +++ b/ffi-cdecl/xz_cdecl.c @@ -0,0 +1,76 @@ +#include + +#include "ffi-cdecl.h" + +// base {{{ + +cdecl_const(LZMA_FILTERS_MAX); +cdecl_const(LZMA_STREAM_HEADER_SIZE); + +cdecl_c99_type(lzma_vli, uint64_t); +cdecl_type(lzma_allocator); +cdecl_type(lzma_bool); +cdecl_type(lzma_reserved_enum); +cdecl_type(lzma_ret); + +// }}} + +/// check {{{ + +cdecl_type(lzma_check); + +cdecl_func(lzma_check_size); + +/// }}} + +/// filter {{{ + +cdecl_type(lzma_filter); + +/// }}} + +/// stream {{{ + +cdecl_type(lzma_stream_flags); + +cdecl_func(lzma_stream_flags_compare); +cdecl_func(lzma_stream_footer_decode); +cdecl_func(lzma_stream_footer_encode); +cdecl_func(lzma_stream_header_decode); +cdecl_func(lzma_stream_header_encode); + +// }}} + +/// index {{{ + +cdecl_type(lzma_index); +cdecl_type(lzma_index_iter); +cdecl_type(lzma_index_iter_mode); + +cdecl_func(lzma_index_append); +cdecl_func(lzma_index_block_count); +cdecl_func(lzma_index_buffer_decode); +cdecl_func(lzma_index_buffer_encode); +cdecl_func(lzma_index_end); +cdecl_func(lzma_index_init); +cdecl_func(lzma_index_iter_init); +cdecl_func(lzma_index_iter_next); +cdecl_func(lzma_index_iter_rewind); +cdecl_func(lzma_index_size); +cdecl_func(lzma_index_stream_count); + +/// }}} + +/// block {{{ + +cdecl_type(lzma_block); + +cdecl_func(lzma_block_buffer_decode); +cdecl_func(lzma_block_compressed_size); +cdecl_func(lzma_block_header_decode); +cdecl_func(lzma_block_total_size); +cdecl_func(lzma_block_unpadded_size); + +/// }}} + +// vim: foldmethod=marker foldlevel=0 diff --git a/ffi/xz_h.lua b/ffi/xz_h.lua new file mode 100644 index 000000000..171ffeeac --- /dev/null +++ b/ffi/xz_h.lua @@ -0,0 +1,174 @@ +-- Automatically generated with ffi-cdecl. + +local ffi = require("ffi") + +ffi.cdef[[ +static const int LZMA_FILTERS_MAX = 4; +static const int LZMA_STREAM_HEADER_SIZE = 12; +typedef uint64_t lzma_vli; +typedef struct { + void *(*alloc)(void *, size_t, size_t); + void (*free)(void *, void *); + void *opaque; +} lzma_allocator; +typedef unsigned char lzma_bool; +typedef enum { + LZMA_RESERVED_ENUM = 0, +} lzma_reserved_enum; +typedef enum { + LZMA_OK = 0, + LZMA_STREAM_END = 1, + LZMA_NO_CHECK = 2, + LZMA_UNSUPPORTED_CHECK = 3, + LZMA_GET_CHECK = 4, + LZMA_MEM_ERROR = 5, + LZMA_MEMLIMIT_ERROR = 6, + LZMA_FORMAT_ERROR = 7, + LZMA_OPTIONS_ERROR = 8, + LZMA_DATA_ERROR = 9, + LZMA_BUF_ERROR = 10, + LZMA_PROG_ERROR = 11, + LZMA_SEEK_NEEDED = 12, + LZMA_RET_INTERNAL1 = 101, + LZMA_RET_INTERNAL2 = 102, + LZMA_RET_INTERNAL3 = 103, + LZMA_RET_INTERNAL4 = 104, + LZMA_RET_INTERNAL5 = 105, + LZMA_RET_INTERNAL6 = 106, + LZMA_RET_INTERNAL7 = 107, + LZMA_RET_INTERNAL8 = 108, +} lzma_ret; +typedef enum { + LZMA_CHECK_NONE = 0, + LZMA_CHECK_CRC32 = 1, + LZMA_CHECK_CRC64 = 4, + LZMA_CHECK_SHA256 = 10, +} lzma_check; +uint32_t lzma_check_size(lzma_check) __attribute__((nothrow, const)); +typedef struct { + lzma_vli id; + void *options; +} lzma_filter; +typedef struct { + uint32_t version; + lzma_vli backward_size; + lzma_check check; + lzma_reserved_enum reserved_enum1; + lzma_reserved_enum reserved_enum2; + lzma_reserved_enum reserved_enum3; + lzma_reserved_enum reserved_enum4; + lzma_bool reserved_bool1; + lzma_bool reserved_bool2; + lzma_bool reserved_bool3; + lzma_bool reserved_bool4; + lzma_bool reserved_bool5; + lzma_bool reserved_bool6; + lzma_bool reserved_bool7; + lzma_bool reserved_bool8; + uint32_t reserved_int1; + uint32_t reserved_int2; +} lzma_stream_flags; +lzma_ret lzma_stream_flags_compare(const lzma_stream_flags *, const lzma_stream_flags *) __attribute__((nothrow, pure)); +lzma_ret lzma_stream_footer_decode(lzma_stream_flags *, const uint8_t *) __attribute__((nothrow)); +lzma_ret lzma_stream_footer_encode(const lzma_stream_flags *, uint8_t *) __attribute__((nothrow)); +lzma_ret lzma_stream_header_decode(lzma_stream_flags *, const uint8_t *) __attribute__((nothrow)); +lzma_ret lzma_stream_header_encode(const lzma_stream_flags *, uint8_t *) __attribute__((nothrow)); +typedef struct lzma_index_s lzma_index; +typedef struct { + struct { + const lzma_stream_flags *flags; + const void *reserved_ptr1; + const void *reserved_ptr2; + const void *reserved_ptr3; + lzma_vli number; + lzma_vli block_count; + lzma_vli compressed_offset; + lzma_vli uncompressed_offset; + lzma_vli compressed_size; + lzma_vli uncompressed_size; + lzma_vli padding; + lzma_vli reserved_vli1; + lzma_vli reserved_vli2; + lzma_vli reserved_vli3; + lzma_vli reserved_vli4; + } stream; + struct { + lzma_vli number_in_file; + lzma_vli compressed_file_offset; + lzma_vli uncompressed_file_offset; + lzma_vli number_in_stream; + lzma_vli compressed_stream_offset; + lzma_vli uncompressed_stream_offset; + lzma_vli uncompressed_size; + lzma_vli unpadded_size; + lzma_vli total_size; + lzma_vli reserved_vli1; + lzma_vli reserved_vli2; + lzma_vli reserved_vli3; + lzma_vli reserved_vli4; + const void *reserved_ptr1; + const void *reserved_ptr2; + const void *reserved_ptr3; + const void *reserved_ptr4; + } block; + union { + const void *p; + size_t s; + lzma_vli v; + } internal[6]; +} lzma_index_iter; +typedef enum { + LZMA_INDEX_ITER_ANY = 0, + LZMA_INDEX_ITER_STREAM = 1, + LZMA_INDEX_ITER_BLOCK = 2, + LZMA_INDEX_ITER_NONEMPTY_BLOCK = 3, +} lzma_index_iter_mode; +lzma_ret lzma_index_append(lzma_index *, const lzma_allocator *, lzma_vli, lzma_vli) __attribute__((nothrow)); +lzma_vli lzma_index_block_count(const lzma_index *) __attribute__((nothrow, pure)); +lzma_ret lzma_index_buffer_decode(lzma_index **, uint64_t *, const lzma_allocator *, const uint8_t *, size_t *, size_t) __attribute__((nothrow)); +lzma_ret lzma_index_buffer_encode(const lzma_index *, uint8_t *, size_t *, size_t) __attribute__((nothrow)); +void lzma_index_end(lzma_index *, const lzma_allocator *) __attribute__((nothrow)); +lzma_index *lzma_index_init(const lzma_allocator *) __attribute__((nothrow)); +void lzma_index_iter_init(lzma_index_iter *, const lzma_index *) __attribute__((nothrow)); +lzma_bool lzma_index_iter_next(lzma_index_iter *, lzma_index_iter_mode) __attribute__((nothrow)); +void lzma_index_iter_rewind(lzma_index_iter *) __attribute__((nothrow)); +lzma_vli lzma_index_size(const lzma_index *) __attribute__((nothrow, pure)); +lzma_vli lzma_index_stream_count(const lzma_index *) __attribute__((nothrow, pure)); +typedef struct { + uint32_t version; + uint32_t header_size; + lzma_check check; + lzma_vli compressed_size; + lzma_vli uncompressed_size; + lzma_filter *filters; + uint8_t raw_check[64]; + void *reserved_ptr1; + void *reserved_ptr2; + void *reserved_ptr3; + uint32_t reserved_int1; + uint32_t reserved_int2; + lzma_vli reserved_int3; + lzma_vli reserved_int4; + lzma_vli reserved_int5; + lzma_vli reserved_int6; + lzma_vli reserved_int7; + lzma_vli reserved_int8; + lzma_reserved_enum reserved_enum1; + lzma_reserved_enum reserved_enum2; + lzma_reserved_enum reserved_enum3; + lzma_reserved_enum reserved_enum4; + lzma_bool ignore_check; + lzma_bool reserved_bool2; + lzma_bool reserved_bool3; + lzma_bool reserved_bool4; + lzma_bool reserved_bool5; + lzma_bool reserved_bool6; + lzma_bool reserved_bool7; + lzma_bool reserved_bool8; +} lzma_block; +lzma_ret lzma_block_buffer_decode(lzma_block *, const lzma_allocator *, const uint8_t *, size_t *, size_t, uint8_t *, size_t *, size_t) __attribute__((nothrow)); +lzma_ret lzma_block_compressed_size(lzma_block *, lzma_vli) __attribute__((nothrow)); +lzma_ret lzma_block_header_decode(lzma_block *, const lzma_allocator *, const uint8_t *) __attribute__((nothrow)); +lzma_vli lzma_block_total_size(const lzma_block *) __attribute__((nothrow, pure)); +lzma_vli lzma_block_unpadded_size(const lzma_block *) __attribute__((nothrow, pure)); +]] From 62831b347266b5430b8a74184a623b6504d3ebdd Mon Sep 17 00:00:00 2001 From: Benoit Pierre Date: Sun, 31 Aug 2025 22:05:55 +0200 Subject: [PATCH 14/17] downloader: new module to HTTP fetch data --- ffi/downloader.lua | 87 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 ffi/downloader.lua diff --git a/ffi/downloader.lua b/ffi/downloader.lua new file mode 100644 index 000000000..03a8994a1 --- /dev/null +++ b/ffi/downloader.lua @@ -0,0 +1,87 @@ +local http = require("socket.http") +local ffi = require("ffi") + +local Downloader = {} + +function Downloader:new() + local o = {} + setmetatable(o, self) + self.__index = self + return o +end + +function Downloader:free() + self.ce:free() + self.ce = nil +end + +local function merge_ranges(ranges) + local new_ranges = {} + for i, r in ipairs(ranges) do + if #new_ranges > 0 and new_ranges[#new_ranges][2] == r[1] - 1 then + new_ranges[#new_ranges][2] = r[2] + else + table.insert(new_ranges, r) + end + end + return new_ranges +end + +function Downloader:fetch(url, callback, ranges, etag, stats) + assert(not (ranges and etag)) + self.status_code = nil + self.etag = nil + local ok + local sink = function(s) + return s and callback(ffi.cast("uint8_t *", s), #s) + end + local body, status_code, resp_headers, status_line + if ranges then + ranges = merge_ranges(ranges) + local range_support_checked = false + local ranges_index = 1 + repeat + body, status_code, resp_headers, status_line = http.request{ + url = url, + headers = { ["Range"] = string.format("bytes=%u-%u", ranges[ranges_index][1], ranges[ranges_index][2]) }, + sink = sink, + } + if not body then + self.err = status_code + return false + end + if not range_support_checked then + if resp_headers["accept-ranges"] ~= "bytes" then + self.err = "server does not support range requests!" + return false + end + range_support_checked = true + end + ok = status_code == 206 + if not ok then + self.err = status_line + return false + end + ranges_index = ranges_index + 1 + until ranges_index > #ranges + else + body, status_code, resp_headers, status_line = http.request{ + url = url, + headers = etag and { ["If-None-Match"] = etag }, + sink = sink, + } + if not body then + self.err = status_code + return false + end + self.etag = resp_headers['etag'] + ok = status_code == 200 or status_code == 304 + if not ok then + self.err = status_line + end + end + self.status_code = status_code + return ok +end + +return Downloader From d1ad173c7fc76994152a34e137289942f35770db Mon Sep 17 00:00:00 2001 From: Benoit Pierre Date: Sun, 31 Aug 2025 22:05:56 +0200 Subject: [PATCH 15/17] luajit: set install prefix to `/usr` So default LUA path / cpath includes the standard (distribution) paths. --- thirdparty/luajit/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/thirdparty/luajit/CMakeLists.txt b/thirdparty/luajit/CMakeLists.txt index d98f8028a..6851ac03f 100644 --- a/thirdparty/luajit/CMakeLists.txt +++ b/thirdparty/luajit/CMakeLists.txt @@ -28,7 +28,7 @@ endif() if(EMULATE_READER) append_autotools_vars(BUILD_ARGS) - list(APPEND BUILD_ARGS TARGET_STRIP=true) + list(APPEND BUILD_ARGS PREFIX=/usr TARGET_STRIP=true) else() assert_var_defined(BASE_CFLAGS) assert_var_defined(HOSTCC) @@ -59,6 +59,7 @@ else() "HOST_CC=${CCACHE} ${HOST_CC}" HOST_CFLAGS=${HOSTCFLAGS} HOST_LDFLAGS= + PREFIX=/usr "TARGET_AR=${AR} rcus" "TARGET_CFLAGS=${CFLAGS} -DLUAJIT_SECURITY_STRHASH=0 -DLUAJIT_SECURITY_STRID=0" TARGET_LDFLAGS=${LDFLAGS} From b89282bac777bb749076c428978e19c6bf78f074 Mon Sep 17 00:00:00 2001 From: Benoit Pierre Date: Sun, 31 Aug 2025 22:05:57 +0200 Subject: [PATCH 16/17] kotasync: new module for OTA --- ffi/kotasync.lua | 738 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 738 insertions(+) create mode 100644 ffi/kotasync.lua diff --git a/ffi/kotasync.lua b/ffi/kotasync.lua new file mode 100644 index 000000000..417246174 --- /dev/null +++ b/ffi/kotasync.lua @@ -0,0 +1,738 @@ +local Hashoir = require("ffi/hashoir") +local band = require("bit").band +local buffer = require("string.buffer") +local ffi = require("ffi") +local json = require("dkjson") +local lfs = require("libs/libkoreader-lfs") +local zstd = require("ffi/zstd") + +require "ffi/loadlib" +require "ffi/posix_h" +require "ffi/xz_h" + +local BLOCK_SIZE = 16 * 1024 +local C = ffi.C + +-- POSIX {{{ + +local function strerror(err) + return ffi.string(C.strerror(err or ffi.errno())) +end + +local function ensure_open(path, flags, mode) + local fd = C.open(path, flags or C.O_RDONLY, mode and ffi.cast("mode_t", mode)) + if fd < 0 then + error("open: "..strerror()) + end + return fd +end + +local function ensure_create(path, mode) + return ensure_open(path, C.O_CREAT + C.O_WRONLY + C.O_TRUNC, mode or (C.S_IRUSR + C.S_IWUSR + C.S_IRGRP + C.S_IROTH)) +end + +local function ensure_lseek(fd, offset, whence) + local ret = C.lseek(fd, offset, whence or C.SEEK_SET) + if ret < 0 then + error("lseek: "..strerror()) + end + return ret +end + +local function ensure_read(fd, ptr, len) + local ret = C.read(fd, ptr, len) + if ret ~= len then + error("read: "..(ret < 0 and strerror() or string.format("short read, %u/%u", ret, len))) + end + return ret +end + +local function ensure_write(fd, ptr, len) + local ret = C.write(fd, ptr, len) + if ret ~= len then + error("write: "..(ret < 0 and strerror() or string.format("short write, %u/%u", ret, len))) + end + return ret +end + +-- }}} + +-- Misc {{{ + +local function read_fd_blocks_iterator(fd, buf, len, block_size) + block_size = block_size or BLOCK_SIZE + local ptr = buf:reserve(block_size) + return function() + assert(len >= 0) + if len == 0 then + return + end + local count = math.min(len, block_size) + ensure_read(fd, ptr, count) + len = len - count + return ptr, count + end +end + +local function hash_file(path, size) + local fd = ensure_open(path) + local buf = buffer:new() + local hashoir = Hashoir:new() + for ptr, len in read_fd_blocks_iterator(fd, buf, size) do + hashoir:update(ptr, len) + end + C.close(fd) + return hashoir:hexdigest() +end + +local function path_matches(path, size, hash) + if lfs.attributes(path, "size") ~= size then + return false + end + local ok, file_hash = pcall(hash_file, path, size) + return ok and file_hash == hash +end + +-- }}} + +-- TAR {{{ + +ffi.cdef[[ +struct ustar_header { + char path[100]; + char mode[8]; + char owner[8]; + char group[8]; + char size[12]; + char mtime[12]; + char checksum[8]; + char type; + char link_name[100]; + char ustar_indicator[6]; + char ustar_version[2]; + char owner_name[32]; + char group_name[32]; + char device_major[8]; + char device_minor[8]; + char path_prefix[155]; +}; +]] + +assert(ffi.sizeof("struct ustar_header") == 500) + +-- "ustar \0" +local FORMAT_GNUTAR = ffi.new("uint8_t[8]", 0x75, 0x73, 0x74, 0x61, 0x72, 0x20, 0x20, 0x00) +-- "ustar\000" +local FORMAT_USTAR = ffi.new("uint8_t[8]", 0x75, 0x73, 0x74, 0x61, 0x72, 0x00, 0x30, 0x30) + +local function tar_size(file_size) + local size = file_size + 511 + return 512 + size - band(size, 511) +end + +-- }}} + +-- Manifest {{{ + +local function deserialize_manifest(ptr, len) + ptr, len = zstd.zstd_uncompress(ptr, len) + local manifest = json.decode(ffi.string(ptr, len)) + C.free(ffi.gc(ptr, nil)) + assert(manifest) + return manifest +end + +local function load_manifest(path) + local len = lfs.attributes(path, "size") + local fd = ensure_open(path) + local buf = buffer:new() + local ptr = buf:reserve(len) + local ret = C.read(fd, ptr, len) + local err = ffi.errno() + C.close(fd) + if ret ~= len then + error(ret < 0 and strerror(err) or string.format("short read, %u/%u", ret, len)) + end + return deserialize_manifest(ptr, len) +end + +local function serialize_manifest(manifest, compression_level) + local data = json.encode(manifest, {keyorder = { + "filename", "files", + "hash", "path", "size", + "xz_check", "xz_hash", "xz_offset", "xz_size", + }}) + assert(data) + local ptr, len = zstd.zstd_compress(data, #data, compression_level or 19) + return ffi.gc(ptr, C.free), len +end + +local function save_manifest(path, manifest, compression_level) + local ptr, len = serialize_manifest(manifest, compression_level) + local fd = ensure_create(path) + ensure_write(fd, ptr, len) + C.free(ffi.gc(ptr, nil)) + C.close(fd) +end + +-- }}} + +-- XZ {{{ + +-- Our indexes are not that big. +local XZ_INDEX_MAXSIZE = 2 * 1024 * 1024 +local XZ_INDEX_MEMLIMIT = 4 * 1024 * 1024 + +-- We compile a static lzma library, linked into +-- our archive library, and use that by default. +local xz +if (os.getenv("KOTASYNC_USE_XZ_LIB") or ""):match(".") then + xz = ffi.loadlib("lzma", "5") +else + xz = ffi.loadlib("archive", "13") +end + +local function xz_block_header_size_decode(ptr) + return (ptr[0] + 1) * 4 +end + +local function xz_padded_size(unpadded_size) + local size = unpadded_size + 3 + return size - band(size, 3) +end + +local function free_xz_index(i) + xz.lzma_index_end(i, nil) +end + +-- }}} + +-- TAR.XZ {{{ + +local TarXz = {} + +function TarXz:new() + local o = {} + setmetatable(o, self) + self.__index = self + return o +end + +function TarXz:close() + if self.tmpfd then + C.close(self.tmpfd) + self.tmpfd = nil + end + if self.fd then + C.close(self.fd) + self.fd = nil + end + return self +end + +function TarXz:open(filename, manifest) + self:close() + self.filename = filename + local fd = ensure_open(filename) + self.fd = fd + local stream_buf = ffi.new("uint8_t[?]", xz.LZMA_STREAM_HEADER_SIZE) + local ret + -- Check header. + ensure_read(fd, stream_buf, xz.LZMA_STREAM_HEADER_SIZE) + local header_stream_flags = ffi.new("lzma_stream_flags") + ret = xz.lzma_stream_header_decode(header_stream_flags, stream_buf) + assert(ret == xz.LZMA_OK, ret) + assert(header_stream_flags.version == 0) + local check_size = xz.lzma_check_size(header_stream_flags.check) + assert(check_size > 0) + -- Check footer. + ensure_lseek(fd, -xz.LZMA_STREAM_HEADER_SIZE, C.SEEK_END) + ensure_read(fd, stream_buf, xz.LZMA_STREAM_HEADER_SIZE) + local footer_stream_flags = ffi.new("lzma_stream_flags") + ret = xz.lzma_stream_footer_decode(footer_stream_flags, stream_buf) + assert(ret == xz.LZMA_OK, ret) + -- Check header & footer match. + ret = xz.lzma_stream_flags_compare(header_stream_flags, footer_stream_flags) + assert(ret == xz.LZMA_OK, ret) + -- Decode index. + ensure_lseek(fd, -(footer_stream_flags.backward_size + xz.LZMA_STREAM_HEADER_SIZE), C.SEEK_END) + local index_buf = ffi.gc(ffi.cast("uint8_t *", C.malloc(footer_stream_flags.backward_size)), C.free) + ensure_read(fd, index_buf, footer_stream_flags.backward_size) + local index = ffi.new("lzma_index *[1]") + local memlimit = ffi.new("uint64_t[1]", XZ_INDEX_MEMLIMIT) + local pos = ffi.new("size_t[1]") + ret = xz.lzma_index_buffer_decode(index, memlimit, nil, index_buf, pos, footer_stream_flags.backward_size) + assert(ret == xz.LZMA_OK, ret) + index = ffi.gc(index[0], free_xz_index) + assert(xz.lzma_index_stream_count(index) == 1) + -- Iterate over blocks. + local entries = {} + local by_path = {} + local manifest_entries = {} + local hashoir = Hashoir:new() + local index_iter = ffi.new("lzma_index_iter") + xz.lzma_index_iter_init(index_iter, index) + while xz.lzma_index_iter_next(index_iter, xz.LZMA_INDEX_ITER_NONEMPTY_BLOCK) == 0 do + -- Read compressed data. + ensure_lseek(fd, index_iter.block.compressed_file_offset) + local comp_size = tonumber(index_iter.block.total_size) + local comp_buf = ffi.new("uint8_t[?]", comp_size) + ensure_read(fd, comp_buf, comp_size) + -- Decode block header. + local block = ffi.new("lzma_block") + block.header_size = xz_block_header_size_decode(comp_buf) + block.check = header_stream_flags.check + local filters = ffi.new("lzma_filter[?]", xz.LZMA_FILTERS_MAX + 1) + block.filters = filters + ret = xz.lzma_block_header_decode(block, nil, comp_buf) + assert(ret == xz.LZMA_OK, ret) + assert(block.uncompressed_size == index_iter.block.uncompressed_size) + ret = xz.lzma_block_compressed_size(block, index_iter.block.unpadded_size) + assert(ret == xz.LZMA_OK) + assert(xz.lzma_block_unpadded_size(block) == index_iter.block.unpadded_size) + assert(xz.lzma_block_total_size(block) == index_iter.block.total_size) + -- Decompress block. + local uncomp_size = block.uncompressed_size + local uncomp_buf = ffi.new("uint8_t[?]", uncomp_size) + local in_pos = ffi.new("size_t[1]", block.header_size) + local out_pos = ffi.new("size_t[1]", 0) + ret = xz.lzma_block_buffer_decode(block, nil, comp_buf, in_pos, comp_size, uncomp_buf, out_pos, uncomp_size) + assert(ret == xz.LZMA_OK, ret) + -- Check TAR header. + local ustar_header = ffi.cast("struct ustar_header *", uncomp_buf) + if ustar_header.path[0] == 0 then + -- Empty final block. + assert(index_iter.block.number_in_file == xz.lzma_index_block_count(index)) + break + end + -- We only support the UStar & GNUtar formats. + -- NOTE: this need to be checked **after** handling the final + -- empty block, whose header is using the original Unix format. + assert(C.memcmp(ustar_header.ustar_indicator, FORMAT_GNUTAR, 8) == 0 or + C.memcmp(ustar_header.ustar_indicator, FORMAT_USTAR, 8) == 0) + assert(ustar_header.path_prefix[0] == 0) + local path = ffi.string(ustar_header.path) + ustar_header.size[11] = 0 + local file_size = tonumber(ffi.string(ustar_header.size, 12), 8) + assert(tar_size(file_size) == block.uncompressed_size) + if manifest == path then + for entry in ffi.string(uncomp_buf + 512, file_size):gmatch("[^\n]+") do + table.insert(manifest_entries, entry) + end + end + local entry = { + hash = file_size ~= 0 and hashoir:reset():update(uncomp_buf + 512, file_size):hexdigest() or nil, + path = path, + size = tonumber(file_size), + xz_hash = hashoir:reset():update(comp_buf, comp_size):hexdigest(), + xz_offset = tonumber(index_iter.block.compressed_file_offset), + xz_size = tonumber(index_iter.block.unpadded_size), + } + table.insert(entries, entry) + by_path[entry.path] = entry + end + if manifest and #manifest_entries == 0 then + error("manifest entry missing or empty") + end + self.header_stream_flags = header_stream_flags + self.footer_stream_flags = footer_stream_flags + self.manifest = manifest_entries + self.entries = entries + self.by_path = by_path + return self +end + +function TarXz:each() + local i = 0 + return function() + i = i + 1 + if i > #self.entries then + return + end + return self.entries[i] + end +end + +function TarXz:reorder(older_tar_xz_or_manifest_path) + local by_path + if older_tar_xz_or_manifest_path:match("[.]kotasync$") then + by_path = {} + for i, v in ipairs(load_manifest(older_tar_xz_or_manifest_path).files) do + by_path[v.path] = v + end + else + local older_tar_xz = TarXz:new():open(older_tar_xz_or_manifest_path) + by_path = older_tar_xz.by_path + older_tar_xz:close() + end + local entries = {} + for i, e in ipairs(self.entries) do + if e.size == 0 then + -- Folder. + e.sort = {2, e.xz_offset} + else + local old = by_path[e.path] + if old and old.size == e.size and old.hash == e.hash then + -- Unmodified file. + e.sort = {1, old.xz_offset} + else + -- New/modified file. + e.sort = {0, e.xz_offset} + end + end + table.insert(entries, e) + end + table.sort(entries, function(e1, e2) + return e1.sort[1] < e2.sort[1] or (e1.sort[1] == e2.sort[1] and e1.sort[2] < e2.sort[2]) + end) + for i, e in ipairs(entries) do + e.sort = nil + end + return self:rewrite(entries) +end + +function TarXz:rewrite(entries) + local directory = self.filename:match("^(.+)/[^/]+$") or "./" + local template_size = #directory + 19 + local template = ffi.new("char[?]", template_size, directory.."/XXXXXX.tar.xz.part") + local fd = C.mkstemps(template, 12) + if fd < 0 then + error("mkstemps: "..strerror()) + end + self.tmpfd = fd + local stream_buf = ffi.new("uint8_t[?]", xz.LZMA_STREAM_HEADER_SIZE) + local ret + -- First: the header. + ret = xz.lzma_stream_header_encode(self.header_stream_flags, stream_buf) + assert(ret == xz.LZMA_OK, ret) + ensure_write(fd, stream_buf, C.LZMA_STREAM_HEADER_SIZE) + -- Second: the blocks. + local offset = C.LZMA_STREAM_HEADER_SIZE + local by_path = {} + local buf = buffer:new() + for i, e in ipairs(entries) do + ensure_lseek(self.fd, e.xz_offset) + e.xz_offset = offset + local size = xz_padded_size(e.xz_size) + for ptr, len in read_fd_blocks_iterator(self.fd, buf, size) do + ensure_write(fd, ptr, len) + end + by_path[e.path] = e + offset = offset + size + end + C.close(self.fd) + self.fd = nil + -- Third: the index. + local index = ffi.gc(xz.lzma_index_init(nil), free_xz_index) + assert(index ~= nil) + for i, e in ipairs(entries) do + ret = xz.lzma_index_append(index, nil, e.xz_size, tar_size(e.size)) + assert(ret == xz.LZMA_OK, ret) + end + assert(xz.lzma_index_block_count(index) == #entries) + local index_size = xz.lzma_index_size(index) + local index_buf = ffi.new("uint8_t[?]", index_size) + local pos = ffi.new("size_t[1]") + ret = xz.lzma_index_buffer_encode(index, index_buf, pos, index_size) + assert(ret == xz.LZMA_OK, ret) + ensure_write(fd, index_buf, index_size) + -- Finally: the footer. + self.footer_stream_flags.backward_size = index_size + ret = xz.lzma_stream_footer_encode(self.footer_stream_flags, stream_buf) + assert(ret == xz.LZMA_OK, ret) + ensure_write(fd, stream_buf, C.LZMA_STREAM_HEADER_SIZE) + -- Rename temporary file and update internal state. + local ok, err = os.rename(ffi.string(template, template_size), self.filename) + if not ok then + error(err) + end + self.entries = entries + self.by_path = by_path + self.fd = fd + self.tmpfd = nil + return self +end + +-- }}} + +-- Updater. {{{ + +local Updater = {} + +function Updater:new(state_dir, dl) + local o = {} + setmetatable(o, self) + self.__index = self + local Downloader = require("ffi/downloader") + o.dl = dl or Downloader:new() + o.own_dl = dl ~= nil + o.state_dir = state_dir + o.update_url = nil + o.manifest = nil + o.missing = nil + o.fd = nil + return o +end + +function Updater:free() + if self.fd then + C.close(self.fd) + self.fd = nil + end + if self.own_dl then + self.dl:free() + end + self.dl = nil + self.missing = nil + self.manifest = nil + self.state_dir = nil + self.update_url = nil +end + +function Updater:fetch_manifest(manifest_url) + local url = require("socket.url") + local url_path = url.parse(manifest_url).path + local basename = table.remove(url.parse_path(url_path)) + local manifest_file = self.state_dir.."/"..basename + local etag_file = manifest_file..".etag" + local manifest, etag + -- Load existing manifest if present in state directory. + if lfs.attributes(manifest_file, "mode") == "file" then + local ok, ret = pcall(load_manifest, manifest_file) + if ok then + manifest = ret + end + end + -- And the associated ETag if applicable. + if manifest and lfs.attributes(etag_file, "mode") == "file" then + etag = io.lines(etag_file)() + end + -- Try fetching an updated version. + local buf = buffer:new() + if not self.dl:fetch(manifest_url, function(ptr, len) + buf:putcdata(ptr, len) + return true + end, nil, etag) then + error(self.dl.err) + end + if self.dl.status_code ~= 304 then -- 304: Not Modified. + local ptr, len = buf:ref() + manifest = deserialize_manifest(ptr, len) + save_manifest(manifest_file, manifest, 3) + -- Update ETag file. + if self.dl.etag then + local fp = io.open(etag_file, "w") + fp:write(self.dl.etag) + fp:close() + else + os.remove(etag_file) + end + end + self.manifest = manifest + self.update_url = url.absolute(manifest_url, manifest.filename) + return manifest.filename +end + +function Updater:prepare_update(seed, progress_cb) + local missing_files = {} + local reused_size = 0 + local download_size = 0 + local matches = function(e) + return false + end + if seed then + if type(seed) == "table" then + matches = function(e) + local se = seed[e.path] + return se and se.size == e.size and se.hash == e.hash + end + else + assert(lfs.attributes(seed, "mode") == "directory") + matches = function(e) + return path_matches(seed.."/"..e.path, e.size, e.hash) + end + end + end + for i, e in ipairs(self.manifest.files) do + if matches(e) then + reused_size = reused_size + e.size + else + download_size = download_size + xz_padded_size(e.xz_size) + table.insert(missing_files, e) + end + if progress_cb and not progress_cb(i) then + -- Canceled. + return + end + end + self.missing_files = missing_files + return { + missing_files = #self.missing_files, + total_files = #self.manifest.files, + reused_size = reused_size, + download_size = download_size + } +end + +function Updater:download_update(progress_cb) + if #self.missing_files == 0 then + -- Nothing to update! + return + end + local update_file = self.state_dir.."/update.tar.xz" + os.remove(update_file) + local fd = ensure_create(update_file..".part") + self.fd = fd + local stream_buf = ffi.new("uint8_t[?]", xz.LZMA_STREAM_HEADER_SIZE) + local ret + -- Write header. + local header_stream_flags = ffi.new("lzma_stream_flags") + header_stream_flags.check = self.manifest.xz_check + ret = xz.lzma_stream_header_encode(header_stream_flags, stream_buf) + assert(ret == xz.LZMA_OK, ret) + ensure_write(fd, stream_buf, C.LZMA_STREAM_HEADER_SIZE) + local missing = self.missing_files + local offset = xz.LZMA_STREAM_HEADER_SIZE + local ranges = {} + for i, e in ipairs(missing) do + e.new_xz_start = offset + local size = xz_padded_size(e.xz_size) + table.insert(ranges, {e.xz_offset, e.xz_offset + size - 1}) + offset = offset + size + end + local index_offset = offset + -- Ensure we have enough disk space. + ret = C.posix_fallocate(fd, 0, offset + XZ_INDEX_MAXSIZE + xz.LZMA_STREAM_HEADER_SIZE) + if ret ~= 0 then + error("posix_fallocate: "..strerror(ret)) + end + -- Fetch & write missing blocks. + local hashoir = Hashoir:new() + local entry_index = 0 + local entry_left = 0 + local entry_path + local entry + local downloaded = 0 + if not self.dl:fetch(self.update_url, function(ptr, len) + downloaded = downloaded + len + while len > 0 do + if not entry then + entry_index = entry_index + 1 + entry = missing[entry_index] + entry_path = entry.path + ensure_lseek(fd, entry.new_xz_start) + entry_left = xz_padded_size(entry.xz_size) + hashoir:reset() + end + local count = math.min(entry_left, len) + hashoir:update(ptr, count) + ensure_write(fd, ptr, count) + entry_left = entry_left - count + assert(entry_left >= 0) + if entry_left == 0 then + if entry.xz_hash ~= hashoir:hexdigest() then + error("corrupted entry: "..entry_path) + end + entry = nil + end + ptr = ptr + count + len = len - count + end + if progress_cb and not progress_cb(downloaded, entry_index, entry_path) then + return false + end + return true + end, ranges) then + return false, self.dl.err + end + -- Write index. + ensure_lseek(fd, index_offset) + local index = ffi.gc(xz.lzma_index_init(nil), free_xz_index) + for i, e in ipairs(missing) do + ret = xz.lzma_index_append(index, nil, e.xz_size, tar_size(e.size)) + assert(ret == xz.LZMA_OK, ret) + end + local index_size = xz.lzma_index_size(index) + local index_buf = ffi.new("uint8_t[?]", index_size) + local pos = ffi.new("size_t[1]") + ret = xz.lzma_index_buffer_encode(index, index_buf, pos, index_size) + assert(ret == xz.LZMA_OK, ret) + ensure_write(fd, index_buf, index_size) + -- Write footer. + local footer_stream_flags = ffi.new("lzma_stream_flags") + footer_stream_flags.backward_size = index_size + footer_stream_flags.check = self.manifest.xz_check + ret = xz.lzma_stream_footer_encode(footer_stream_flags, stream_buf) + assert(ret == xz.LZMA_OK, ret) + ensure_write(fd, stream_buf, C.LZMA_STREAM_HEADER_SIZE) + -- Finalize. + ret = C.ftruncate(fd, ensure_lseek(fd, 0, C.SEEK_CUR)) + if ret ~= 0 then + error("ftruncate: "..strerror()) + end + C.close(fd) + self.fd = nil + local ok, err = os.rename(update_file..".part", update_file) + if not ok then + error(err) + end +end + +function Updater:download_update_in_subprocess(progress_cb, progress_frequency) + assert(progress_cb and progress_frequency) + local util = require("ffi/util") + local child, read_fd = util.runInSubProcess(function(pid, write_fd) + local last_update = 0 + local msg_len = ffi.new('uint16_t[1]') + local buf = buffer:new() + local send = function(...) + buf:reset():encode{...} + msg_len[0] = #buf + ensure_write(write_fd, msg_len, 2) + local ptr = buf:ref() + ensure_write(write_fd, ptr, msg_len[0]) + end + local ok, err = pcall(self.download_update, self, function(size, count, path) + local new_update = util.getTimestamp() + if new_update - last_update >= progress_frequency then + last_update = new_update + send(false, size, count, path) + end + return true + end) + send(true, ok, err) + end, true) + if not child then + -- read_fd: error message + return child, read_fd + end + local msg_len = ffi.new('uint16_t[1]') + local buf = buffer:new() + while true do + ensure_read(read_fd, msg_len, 2) + local ptr = buf:reserve(msg_len[0]) + ensure_read(read_fd, ptr, msg_len[0]) + buf:commit(msg_len[0]) + local msg = buf:decode() + if msg[1] then + -- It's done. + util.isSubProcessDone(child, true) + return msg[2], msg[3] + end + if not progress_cb(msg[2], msg[3], msg[4]) then + -- Canceled. + C.kill(-child, 15) + util.isSubProcessDone(child, true) + return false + end + end +end + +-- }}} + +return { + load_manifest = load_manifest, + save_manifest = save_manifest, + TarXz = TarXz, + Updater = Updater, +} From caa6030e3d4a826f8a4880208e999211f34785b4 Mon Sep 17 00:00:00 2001 From: Benoit Pierre Date: Sun, 31 Aug 2025 22:05:58 +0200 Subject: [PATCH 17/17] kotasync: add command line script + binary launcher --- kotasync/kotasync.c | 50 ++++ kotasync/kotasync.lua | 229 ++++++++++++++++++ .../cmake_modules/koreader_targets.cmake | 87 +++++++ .../koreader_thirdparty_libs.cmake | 1 + 4 files changed, 367 insertions(+) create mode 100644 kotasync/kotasync.c create mode 100755 kotasync/kotasync.lua diff --git a/kotasync/kotasync.c b/kotasync/kotasync.c new file mode 100644 index 000000000..c1d1d4207 --- /dev/null +++ b/kotasync/kotasync.c @@ -0,0 +1,50 @@ +#include +#include + +#include +#include + +static const char *bootstrap_script = (R""""( +local redirects = { ["libs/libkoreader-lfs"] = "lfs" } +for modulename in (')"""" LUA_MODULES R""""('):gmatch("[^ ]+") do + local redir = modulename:gsub("/", "_") + if redir ~= modulename then + redirects[modulename] = redir + end +end +table.insert(package.loaders, function (modulename) + local redir = redirects[modulename] + if redir then + return function() return require(redir) end + end +end) +require "kotasync" +)""""); + + +int main(int argc, char *argv[]) { + lua_State *L; + if (setenv("KOTASYNC_USE_XZ_LIB", "1", 0)) { + perror("setenv"); + return 1; + } + L = lua_open(); + if (!L) + return 1; + // Setup `arg` table. + lua_createtable(L, argc, 0); + for (int i = 0; i < argc; ++i) { + lua_pushstring(L, argv[i]); + lua_rawseti(L, -2, i); + } + lua_setglobal(L, "arg"); + // Load standard library. + luaL_openlibs(L); + // And run bootstrap script. + if (luaL_dostring(L, bootstrap_script)) { + const char *msg = lua_tostring(L, -1); + fprintf(stderr, "%s\n", msg); + return EXIT_FAILURE; + } + return EXIT_SUCCESS; +} diff --git a/kotasync/kotasync.lua b/kotasync/kotasync.lua new file mode 100755 index 000000000..313fc7354 --- /dev/null +++ b/kotasync/kotasync.lua @@ -0,0 +1,229 @@ +#!./luajit + +local lfs +local util +local kotasync + +local function naturalsize(size) + local chunk, unit = 1, 'B ' + if size >= 1000*1000*1000 then + chunk, unit = 1000*1000*1000, 'GB' + elseif size >= 1000*1000 then + chunk, unit = 1000*1000, 'MB' + elseif size >= 1000 then + chunk, unit = 1000, 'KB' + end + local fmt = chunk > 1 and "%.1f" or "%u" + return string.format(fmt.." %s", size / chunk, unit) +end + +local function make(tar_xz_path, kotasync_path, tar_xz_manifest, older_tar_xz_or_kotasync_path) + local tar_xz = kotasync.TarXz:new():open(tar_xz_path, tar_xz_manifest) + if older_tar_xz_or_kotasync_path then + tar_xz:reorder(older_tar_xz_or_kotasync_path) + end + local files = {} + local manifest_by_path = tar_xz.by_path + if tar_xz_manifest then + manifest_by_path = {} + for __, f in ipairs(tar_xz.manifest) do + assert(not manifest_by_path[f]) + manifest_by_path[f] = true + end + end + for e in tar_xz:each() do + -- Ignore directories. + if e.size ~= 0 and manifest_by_path[e.path] then + table.insert(files, e) + end + end + if tar_xz_manifest and #files ~= #tar_xz.manifest then + error("mismatched manifest / archive contents") + end + local manifest = { + filename = tar_xz_path:match("([^/]+)$"), + files = files, + xz_check = tonumber(tar_xz.header_stream_flags.check), + } + tar_xz:close() + if not kotasync_path then + assert(tar_xz_path:match("[.]tar.xz$")) + kotasync_path = tar_xz_path:sub(1, -7).."kotasync" + end + kotasync.save_manifest(kotasync_path, manifest) +end + +local function sync(state_dir, manifest_url, seed) + local updater = kotasync.Updater:new(state_dir) + if seed and lfs.attributes(seed, "mode") == "file" then + -- If the seed is a kotasync file, we need to load it + -- now, as it may get overwritten by `fetch_manifest`. + local by_path = {} + for i, e in ipairs(kotasync.load_manifest(seed).files) do + by_path[e.path] = e + end + seed = by_path + end + updater:fetch_manifest(manifest_url) + local total_files = #updater.manifest.files + local last_update = 0 + local delay = false --190000 + local update_frequency = 0.2 + local stats = updater:prepare_update(seed, function(count) + local new_update = util.getTimestamp() + if count ~= total_files and new_update - last_update < update_frequency then + return true + end + last_update = new_update + io.stderr:write(string.format("\ranalyzing: %4u/%4u", count, total_files)) + if delay then + util.usleep(delay) + end + return true + end) + io.stderr:write(string.format("\r%99s\r", "")) + assert(total_files == stats.total_files) + if stats.missing_files == 0 then + print('nothing to update!') + return + end + print(string.format("missing : %u/%u files", stats.missing_files, total_files)) + print(string.format("reusing : %7s (%10u)", naturalsize(stats.reused_size), stats.reused_size)) + print(string.format("fetching: %7s (%10u)", naturalsize(stats.download_size), stats.download_size)) + io.stdout:flush() + local pbar_indicators = {" ", "▏", "▎", "▍", "▌", "▋", "▊", "▉", "█"} + local pbar_size = 16 + local pbar_chunk = (stats.download_size + pbar_size - 1) / pbar_size + local prev_path = "" + local old_progress + last_update = 0 + local ok, err = pcall(updater.download_update, updater, function(size, count, path) + local new_update = util.getTimestamp() + if size ~= stats.download_size and new_update - last_update < update_frequency then + return true + end + last_update = new_update + local padding = math.max(#prev_path, #path) + local progress = math.floor(size / pbar_chunk) + local pbar = pbar_indicators[#pbar_indicators]:rep(progress)..pbar_indicators[1 + math.floor(size % pbar_chunk * #pbar_indicators / pbar_chunk)]..(" "):rep(pbar_size - progress - 1) + local new_progress = string.format("\rdownloading: %8s %4u/%4u %s %-"..padding.."s", size, count, stats.missing_files, pbar, path) + if new_progress ~= old_progress then + old_progress = new_progress + io.stderr:write(new_progress) + end + prev_path = path + if delay then + util.usleep(delay) + end + return true + end) + io.stderr:write(string.format("\r%99s\r", "")) + if not ok then + io.stderr:write(string.format("ERROR: %s", err)) + return 1 + end +end + +local help = [[ +USAGE: kotasync make [-h] [--manifest TAR_XZ_MANIFEST] [--reorder OLDER_TAR_XZ_OR_KOTASYNC_FILE] TAR_XZ_FILE [KOTASYNC_FILE] + kotasync sync [-h] STATE_DIR KOTASYNC_URL [SEED_DIR_OR_KOTASYNC_FILE] + +options: + -h, --help show this help message and exit + +MAKE: + + TAR_XZ_FILE source tar.xz file + KOTASYNC_FILE destination kotasync file + + -m, --manifest TAR_XZ_MANIFEST + archive entry to use as base for manifest + + -r, --reorder OLDER_TAR_XZ_OR_KOTASYNC_FILE + will repack the new tar.xz with this order: + ┌────────────────────┬──────────────────┬─────────────┐ + │ new/modified files │ unmodified files │ folders │ + │ (new order) │ (old order) │ (new order) │ + └────────────────────┴──────────────────┴─────────────┘ +SYNC: + + STATE_DIR destination for the kotasync and update files + KOTASYNC_URL URL of kotasync file + SEED_DIR_OR_KOTASYNC_FILE + optional seed directory / kotasync file +]] + +local function main() + local command + local options = {} + local arguments = {} + while #arg > 0 do + local a = table.remove(arg, 1) + -- print(i, a) + if a:match("^-(.+)$") then + -- print('option', a) + if a == "-h" or a == "--help" then + io.stdout:write(help) + return + elseif command == "make" and (a == "-m" or a == "--manifest") then + if #arg == 0 then + io.stderr:write(string.format("ERROR: option --manifest: expected one argument\n")) + return 2 + end + options.manifest = table.remove(arg, 1) + elseif command == "make" and (a == "-r" or a == "--reorder") then + if #arg == 0 then + io.stderr:write(string.format("ERROR: option --reorder: expected one argument\n")) + return 2 + end + options.reorder = table.remove(arg, 1) + else + io.stderr:write(string.format("ERROR: unrecognized option: %s\n", a)) + return 2 + end + elseif command then + table.insert(arguments, a) + else + command = a + end + end + local fn + if command == "make" then + if #arguments < 1 then + io.stderr:write("ERROR: not enough arguments\n") + return 2 + end + if #arguments > 2 then + io.stderr:write("ERROR: too many arguments\n") + return 2 + end + fn = function() make(arguments[1], arguments[2], options.manifest, options.reorder) end + elseif command == "sync" then + if #arguments < 2 then + io.stderr:write("ERROR: not enough arguments\n") + return 2 + end + if #arguments > 3 then + io.stderr:write("ERROR: too many arguments\n") + return 2 + end + fn = function() sync(arguments[1], arguments[2], arguments[3]) end + elseif not command then + io.stderr:write(help) + return 2 + else + io.stderr:write(string.format("ERROR: unrecognized command: %s\n", command)) + return 2 + end + require("ffi/loadlib") + lfs = require("libs/libkoreader-lfs") + util = require("ffi/util") + kotasync = require("ffi/kotasync") + local ok, err = xpcall(fn, debug.traceback) + if not ok then + io.stderr:write(string.format("ERROR: %s\n", err)) + return 3 + end +end + +os.exit(main()) diff --git a/thirdparty/cmake_modules/koreader_targets.cmake b/thirdparty/cmake_modules/koreader_targets.cmake index b1a4c0e95..1f246d5c0 100644 --- a/thirdparty/cmake_modules/koreader_targets.cmake +++ b/thirdparty/cmake_modules/koreader_targets.cmake @@ -245,6 +245,93 @@ function(setup_osx_loader) set_target_properties(osx_loader PROPERTIES OUTPUT_NAME koreader) endfunction() +# kotasync +declare_koreader_target( + kotasync TYPE executable + DEPENDS luajit::luajit_static + EXCLUDE_FROM_ALL + SOURCES kotasync/kotasync.c +) +function(setup_kotasync) + if(EMULATE_READER) + set(LUAJIT_EXE ${STAGING_DIR}/bin/luajit) + set(LUAJIT_JIT_OPTS) + set(LUA_FFI_POSIX_TYPES posix_types_x64_h) + else() + find_program( + LUAJIT_EXE luajit REQUIRED + PATHS /usr/local/bin /usr/bin + NO_DEFAULT_PATH NO_PACKAGE_ROOT_PATH NO_CMAKE_PATH + NO_CMAKE_ENVIRONMENT_PATH NO_SYSTEM_ENVIRONMENT_PATH + NO_CMAKE_SYSTEM_PATH NO_CMAKE_INSTALL_PREFIX NO_CMAKE_FIND_ROOT_PATH + ) + endif() + set(LUAJIT_JIT_OPTS -d -g -o Linux) + if(CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64") + list(APPEND LUAJIT_JIT_OPTS -X -a arm64) + elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL "arm") + list(APPEND LUAJIT_JIT_OPTS -W -a arm) + elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL "i686") + list(APPEND LUAJIT_JIT_OPTS -W -a x86) + elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") + list(APPEND LUAJIT_JIT_OPTS -X -a x64) + endif() + set(LUA_MODULES_NAMES) + set(LUA_MODULES_OBJS) + foreach(NAME + # cdecls + ffi/posix_types_64b_h + ffi/posix_types_def_h + ffi/posix_types_x64_h + ffi/posix_types_x86_h + ffi/posix_h + ffi/xxhash_h + ffi/xz_h + ffi/zstd_h + # others + ffi/loadlib + ffi/downloader + ffi/hashoir + ffi/util + ffi/zstd + ffi/kotasync + kotasync + ) + string(REPLACE "/" "_" ID ${NAME}) + set(OBJ ${CMAKE_CURRENT_BINARY_DIR}/lua_${ID}.o) + set(SRC ${NAME}.lua) + if(NAME STREQUAL "kotasync") + set(SRC ${CMAKE_CURRENT_SOURCE_DIR}/kotasync/${SRC}) + endif() + add_custom_command( + COMMAND ${LUAJIT_EXE} -b ${LUAJIT_JIT_OPTS} -n ${ID} ${SRC} ${OBJ} + DEPENDS ${SRC} + OUTPUT ${OBJ} + WORKING_DIRECTORY ${OUTPUT_DIR} + VERBATIM + ) + list(APPEND LUA_MODULES_NAMES ${NAME}) + list(APPEND LUA_MODULES_OBJS ${OBJ}) + endforeach() + list(JOIN LUA_MODULES_NAMES " " LUA_MODULES_NAMES) + set_target_properties(kotasync PROPERTIES ENABLE_EXPORTS TRUE) + target_compile_definitions(kotasync PRIVATE "LUA_MODULES=\"${LUA_MODULES_NAMES}\"") + target_compile_options(kotasync BEFORE PRIVATE -std=gnu11) + target_sources(kotasync PRIVATE ${LUA_MODULES_OBJS}) + set(KOTASYNC_SCRIPT ${OUTPUT_DIR}/kotasync.lua) + add_custom_command( + COMMAND ln -snfr ${CMAKE_CURRENT_SOURCE_DIR}/kotasync/kotasync.lua ${KOTASYNC_SCRIPT} + OUTPUT ${KOTASYNC_SCRIPT} + VERBATIM + ) + if(ANDROID OR APPLE OR EMULATE_READER) + set(ALL) + else() + set(ALL ALL) + endif() + add_custom_target(kotasync-script ${ALL} DEPENDS ${KOTASYNC_SCRIPT}) +endfunction() + # inkview-compat if(POCKETBOOK) set(EXCLUDE_FROM_ALL) diff --git a/thirdparty/cmake_modules/koreader_thirdparty_libs.cmake b/thirdparty/cmake_modules/koreader_thirdparty_libs.cmake index 210b6ab24..09161af47 100644 --- a/thirdparty/cmake_modules/koreader_thirdparty_libs.cmake +++ b/thirdparty/cmake_modules/koreader_thirdparty_libs.cmake @@ -164,6 +164,7 @@ else() set(LUAJIT_LIB) endif() get_target_property(LUAJIT_INC luajit::luajit INTERFACE_INCLUDE_DIRECTORIES) +declare_dependency(luajit::luajit_static INCLUDES luajit-2.1 STATIC luajit-5.1 LIBRARIES dl m) # luasec if(MONOLIBTIC)