From c5da028df443b2016757a8835d8954d2783d8d9f Mon Sep 17 00:00:00 2001 From: Alex Gaetano Padula Date: Wed, 14 Oct 2026 14:28:19 -0400 Subject: [PATCH 1/2] issue #43 extend binding to tidesdb 10 (specifically 10.2.1); include extended documentation in repo following org standard --- .github/workflows/build_and_test.yml | 372 ++-- .github/workflows/build_from_source.yml | 390 ----- .gitignore | 37 +- .gitmodules | 3 + Cargo.toml | 39 +- README.md | 26 +- build.rs | 613 ++++--- doc/manual.json | 19 + doc/rust.md | 1596 +++++++++++++++++ scripts/sync-versions.py | 1215 ------------- src/config.rs | 1226 ++++--------- src/db.rs | 1340 ++++++--------- src/error.rs | 138 +- src/ffi.rs | 879 +++++----- src/iterator.rs | 207 +-- src/lib.rs | 2078 +---------------------- src/stats.rs | 527 ++++-- src/tests.rs | 1651 ++++++++++++++++++ src/transaction.rs | 556 ++++-- tidesdb | 1 + 20 files changed, 6099 insertions(+), 6814 deletions(-) delete mode 100644 .github/workflows/build_from_source.yml create mode 100644 .gitmodules create mode 100644 doc/manual.json create mode 100644 doc/rust.md delete mode 100755 scripts/sync-versions.py create mode 100644 src/tests.rs create mode 160000 tidesdb diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 964d9fc..3933992 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -7,8 +7,51 @@ on: branches: [ "master" ] workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +# The TidesDB C source is not downloaded separately: it is the tidesdb/ git submodule, pinned +# to the release the bindings target (build.rs refuses any other), and `submodules: true` +# checks out exactly that commit. + jobs: - build-and-test: + # Every job builds on current stable Rust, except `msrv`, which checks that the minimum + # declared in Cargo.toml (rust-version, which edition 2024 sets at 1.85) still builds. + + # Formatting, lints and rustdoc. + lint: + runs-on: ubuntu-latest + name: fmt / clippy / doc + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - run: sudo apt-get update && sudo apt-get install -y build-essential cmake + - uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt, clippy + - run: cargo fmt --check + - run: cargo clippy --all-targets -- -D warnings + - run: cargo doc --no-deps + env: + RUSTDOCFLAGS: -D warnings + + # The minimum supported Rust version declared in Cargo.toml. + msrv: + runs-on: ubuntu-latest + name: MSRV (1.85) + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - run: sudo apt-get update && sudo apt-get install -y build-essential cmake + - uses: dtolnay/rust-toolchain@1.85.0 + - run: cargo test --release + + # Default build: the bundled TidesDB source built by build.rs, no compression and no + # system libraries. + bundled: strategy: fail-fast: false matrix: @@ -18,55 +61,21 @@ jobs: - os: ubuntu-24.04-arm name: Linux ARM64 - os: macos-latest - name: macOS x64 + name: macOS - os: windows-latest name: Windows x64 runs-on: ${{ matrix.os }} - name: ${{ matrix.name }} + name: bundled / ${{ matrix.name }} steps: - - name: Checkout tidesdb-rs repo - uses: actions/checkout@v4 - with: - path: tidesdb-rs - - - name: Prepare local source crate patches - shell: bash - working-directory: tidesdb-rs - run: | - PYTHON=python3 - if ! command -v python3 >/dev/null 2>&1; then - PYTHON=python - fi - "$PYTHON" -m venv .cargo/sync-venv - VENV_PYTHON=.cargo/sync-venv/bin/python - if [ ! -x "$VENV_PYTHON" ]; then - VENV_PYTHON=.cargo/sync-venv/Scripts/python.exe - fi - "$VENV_PYTHON" -m pip install tomlkit requests InquirerPy - "$VENV_PYTHON" scripts/sync-versions.py \ - --prepare-local-patches .cargo/tidesdb-src \ - --write-cargo-patch-config .cargo/config.toml \ - --versions current \ - --yes - - - name: Checkout tidesdb repo - uses: actions/checkout@v4 + - uses: actions/checkout@v4 with: - repository: tidesdb/tidesdb - ref: v9.3.15 - path: tidesdb + submodules: true - - name: Install dependencies (Linux) + - name: Install build tools (Linux) if: runner.os == 'Linux' - run: | - sudo apt update - sudo apt install -y libzstd-dev liblz4-dev libsnappy-dev build-essential cmake pkg-config - - - name: Install dependencies (macOS) - if: runner.os == 'macOS' - run: brew install zstd lz4 snappy + run: sudo apt-get update && sudo apt-get install -y build-essential cmake - name: Setup MSYS2 (Windows) if: runner.os == 'Windows' @@ -78,127 +87,192 @@ jobs: mingw-w64-x86_64-gcc mingw-w64-x86_64-cmake mingw-w64-x86_64-make - mingw-w64-x86_64-zstd - mingw-w64-x86_64-lz4 - mingw-w64-x86_64-snappy mingw-w64-x86_64-rust - - name: Configure and build TidesDB (Linux) - if: runner.os == 'Linux' - run: | - cd tidesdb - cmake -S . -B build -DTIDESDB_BUILD_TESTS=OFF -DTIDESDB_WITH_SANITIZER=OFF - cmake --build build --config Release - sudo cmake --install build - sudo ldconfig + - name: Install Rust toolchain + if: runner.os != 'Windows' + uses: dtolnay/rust-toolchain@stable - - name: Configure and build TidesDB (macOS) - if: runner.os == 'macOS' - run: | - cd tidesdb - export HOMEBREW_PREFIX=$(brew --prefix) - cmake -S . -B build \ - -DTIDESDB_BUILD_TESTS=OFF \ - -DTIDESDB_WITH_SANITIZER=OFF \ - -DCMAKE_PREFIX_PATH="${HOMEBREW_PREFIX}" - cmake --build build --config Release - sudo cmake --install build - - - name: Create CMake config files for MSYS2 packages (Windows) - if: runner.os == 'Windows' - shell: msys2 {0} + - name: Build and test + if: runner.os != 'Windows' run: | - MINGW_PREFIX_WIN=$(cygpath -m /mingw64) - - mkdir -p /mingw64/lib/cmake/lz4 - mkdir -p /mingw64/lib/cmake/zstd - mkdir -p /mingw64/lib/cmake/Snappy - mkdir -p /mingw64/lib/cmake/PThreads4W - - cat > /mingw64/lib/cmake/lz4/lz4-config.cmake << EOF - if(NOT TARGET lz4::lz4) - add_library(lz4::lz4 SHARED IMPORTED) - set_target_properties(lz4::lz4 PROPERTIES - IMPORTED_LOCATION "${MINGW_PREFIX_WIN}/bin/liblz4.dll" - IMPORTED_IMPLIB "${MINGW_PREFIX_WIN}/lib/liblz4.dll.a" - INTERFACE_INCLUDE_DIRECTORIES "${MINGW_PREFIX_WIN}/include" - ) - endif() - EOF - - cat > /mingw64/lib/cmake/zstd/zstd-config.cmake << EOF - if(NOT TARGET zstd::libzstd_shared) - add_library(zstd::libzstd_shared SHARED IMPORTED) - set_target_properties(zstd::libzstd_shared PROPERTIES - IMPORTED_LOCATION "${MINGW_PREFIX_WIN}/bin/libzstd.dll" - IMPORTED_IMPLIB "${MINGW_PREFIX_WIN}/lib/libzstd.dll.a" - INTERFACE_INCLUDE_DIRECTORIES "${MINGW_PREFIX_WIN}/include" - ) - endif() - EOF - - cat > /mingw64/lib/cmake/Snappy/Snappy-config.cmake << EOF - if(NOT TARGET Snappy::snappy) - add_library(Snappy::snappy SHARED IMPORTED) - set_target_properties(Snappy::snappy PROPERTIES - IMPORTED_LOCATION "${MINGW_PREFIX_WIN}/bin/libsnappy.dll" - IMPORTED_IMPLIB "${MINGW_PREFIX_WIN}/lib/libsnappy.dll.a" - INTERFACE_INCLUDE_DIRECTORIES "${MINGW_PREFIX_WIN}/include" - ) - endif() - EOF - - cat > /mingw64/lib/cmake/PThreads4W/PThreads4W-config.cmake << EOF - if(NOT TARGET PThreads4W::PThreads4W) - add_library(PThreads4W::PThreads4W SHARED IMPORTED) - set_target_properties(PThreads4W::PThreads4W PROPERTIES - IMPORTED_LOCATION "${MINGW_PREFIX_WIN}/bin/libwinpthread-1.dll" - IMPORTED_IMPLIB "${MINGW_PREFIX_WIN}/lib/libpthread.dll.a" - INTERFACE_INCLUDE_DIRECTORIES "${MINGW_PREFIX_WIN}/include" - ) - endif() - EOF - - - name: Configure and build TidesDB (Windows) + cargo build --release + cargo test --release + + # MinGW toolchain from MSYS2. TidesDB's own CI builds MinGW with "MinGW Makefiles"; + # without this the cmake crate picks "MSYS Makefiles", which needs a `make` that the + # mingw-w64 make package (mingw32-make) does not provide. + - name: Build and test (Windows) if: runner.os == 'Windows' shell: msys2 {0} + env: + CMAKE_GENERATOR: MinGW Makefiles run: | - cd tidesdb - cmake -G "MinGW Makefiles" \ - -DCMAKE_MAKE_PROGRAM=mingw32-make \ - -DCMAKE_PREFIX_PATH=/mingw64 \ - -DTIDESDB_WITH_SANITIZER=OFF \ - -DTIDESDB_BUILD_TESTS=OFF \ - -S . -B build - cmake --build build --config Release - cmake --install build --prefix /mingw64 + export PATH="/mingw64/bin:$PATH" + cargo build --release + cargo test --release - - name: Install Rust toolchain - if: runner.os != 'Windows' - uses: dtolnay/rust-toolchain@stable + # Opt-in codecs and allocators, each linking its system library. + features: + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + features: compression + packages: libzstd-dev liblz4-dev libsnappy-dev + - os: ubuntu-latest + features: lz4 + packages: liblz4-dev + - os: ubuntu-latest + features: zstd + packages: libzstd-dev + - os: ubuntu-latest + features: snappy + packages: libsnappy-dev + - os: macos-latest + features: compression + packages: zstd lz4 snappy + - os: ubuntu-latest + features: jemalloc + packages: libjemalloc-dev + - os: ubuntu-latest + features: mimalloc + packages: libmimalloc-dev + - os: ubuntu-latest + features: tcmalloc + packages: libgoogle-perftools-dev + + runs-on: ${{ matrix.os }} + name: features=${{ matrix.features }} / ${{ matrix.os }} + + steps: + - uses: actions/checkout@v4 with: - toolchain: 1.85.0 + submodules: true - - name: Build and test (Linux) + - name: Install dependencies (Linux) if: runner.os == 'Linux' - run: | - cd tidesdb-rs - cargo build --release - cargo test --release -- --test-threads=1 + run: sudo apt-get update && sudo apt-get install -y build-essential cmake pkg-config ${{ matrix.packages }} - - name: Build and test (macOS) + - name: Install dependencies (macOS) if: runner.os == 'macOS' + run: brew install ${{ matrix.packages }} + + - uses: dtolnay/rust-toolchain@stable + + - name: Build and test + run: cargo test --release --features ${{ matrix.features }} + + # A system TidesDB 10.x is linked through pkg-config instead of the bundled source, and one + # of another major is ignored with a warning. TidesDB's CMake install ships no .pc file + # (distro packages may), so the job writes one the way a package would. + system-install: + runs-on: ubuntu-latest + name: pkg-config system install + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@v4 + with: + submodules: true + + - name: Install build tools + run: sudo apt-get update && sudo apt-get install -y build-essential cmake pkg-config + + - name: Build and install TidesDB from the submodule run: | - cd tidesdb-rs - export DYLD_LIBRARY_PATH=/usr/local/lib:$DYLD_LIBRARY_PATH - cargo build --release - cargo test --release -- --test-threads=1 + cmake -S tidesdb -B tidesdb-build -DTIDESDB_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release \ + -DTIDESDB_WITH_LZ4=OFF -DTIDESDB_WITH_ZSTD=OFF -DTIDESDB_WITH_SNAPPY=OFF + cmake --build tidesdb-build + sudo cmake --install tidesdb-build + sudo ldconfig + sudo mkdir -p /usr/local/lib/pkgconfig + printf '%s\n' \ + 'prefix=/usr/local' \ + 'libdir=${prefix}/lib' \ + 'includedir=${prefix}/include' \ + 'Name: tidesdb' \ + 'Description: TidesDB storage engine' \ + 'Version: 10.2.1' \ + 'Libs: -L${libdir} -ltidesdb -lpthread' \ + 'Cflags: -I${includedir}' | sudo tee /usr/local/lib/pkgconfig/tidesdb.pc + pkg-config --modversion tidesdb - - name: Build and test (Windows) - if: runner.os == 'Windows' - shell: msys2 {0} + - uses: dtolnay/rust-toolchain@stable + + # build.rs prints `cargo:rustc-cfg=tidesdb_system` when it links the system library and + # `cargo:rustc-link-lib=static=tidesdb` when it builds the bundled source instead + - name: Build and test against the system library run: | - export PATH="/mingw64/bin:$PATH" - cd tidesdb-rs - cargo build --release - cargo test --release -- --test-threads=1 + cargo build --release -vv 2>&1 | tee build.log + grep -q "cargo:rustc-cfg=tidesdb_system" build.log + if grep -q "cargo:rustc-link-lib=static=tidesdb" build.log; then + echo "expected the system TidesDB to be linked, but the bundled source was built" + exit 1 + fi + cargo test --release + + - name: A system TidesDB of another major is ignored + run: | + sudo sed -i 's/^Version: .*/Version: 11.0.0/' /usr/local/lib/pkgconfig/tidesdb.pc + cargo build --release -vv 2>&1 | tee build.log + grep -q "Ignoring the system TidesDB 11.0.0" build.log + grep -q "cargo:rustc-link-lib=static=tidesdb" build.log + cargo test --release test_linked_version + + # Cross-compiled targets. armv7 is a 32-bit embedded Linux target (it needs libatomic) and + # its tests run under QEMU; Windows GNU from Linux checks that nothing in build.rs depends on + # the build machine. + cross: + strategy: + fail-fast: false + matrix: + include: + - target: armv7-unknown-linux-gnueabihf + packages: gcc-arm-linux-gnueabihf libc6-dev-armhf-cross qemu-user + cc: arm-linux-gnueabihf-gcc + runner: qemu-arm -L /usr/arm-linux-gnueabihf + command: test + - target: aarch64-unknown-linux-gnu + packages: gcc-aarch64-linux-gnu libc6-dev-arm64-cross qemu-user + cc: aarch64-linux-gnu-gcc + runner: qemu-aarch64 -L /usr/aarch64-linux-gnu + command: test + - target: x86_64-pc-windows-gnu + packages: gcc-mingw-w64-x86-64 + cc: x86_64-w64-mingw32-gcc + runner: "" + command: test --no-run + + runs-on: ubuntu-latest + name: cross / ${{ matrix.target }} + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - run: sudo apt-get update && sudo apt-get install -y build-essential cmake ${{ matrix.packages }} + - uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.target }} + - name: Build and test + shell: bash + run: | + triple=$(echo "${{ matrix.target }}" | tr '[:lower:]-' '[:upper:]_') + export "CC_$(echo "${{ matrix.target }}" | tr '-' '_')=${{ matrix.cc }}" + export "CARGO_TARGET_${triple}_LINKER=${{ matrix.cc }}" + if [ -n "${{ matrix.runner }}" ]; then export "CARGO_TARGET_${triple}_RUNNER=${{ matrix.runner }}"; fi + cargo ${{ matrix.command }} --release --target ${{ matrix.target }} + + package: + runs-on: ubuntu-latest + name: cargo package + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - run: sudo apt-get update && sudo apt-get install -y build-essential cmake + - uses: dtolnay/rust-toolchain@stable + # builds the packaged crate from its own tarball, proving the bundled source is complete + - run: cargo package --allow-dirty diff --git a/.github/workflows/build_from_source.yml b/.github/workflows/build_from_source.yml deleted file mode 100644 index 0da93a8..0000000 --- a/.github/workflows/build_from_source.yml +++ /dev/null @@ -1,390 +0,0 @@ -name: Build TidesDB from source - -on: - push: - branches: [ "master" ] - pull_request: - branches: [ "master" ] - workflow_dispatch: - -jobs: - build-from-source: - strategy: - fail-fast: false - matrix: - include: - - os: ubuntu-latest - name: Linux x64 - - os: ubuntu-24.04-arm - name: Linux ARM64 - - os: macos-latest - name: macOS x64 - - os: windows-latest - name: Windows x64 - - runs-on: ${{ matrix.os }} - name: ${{ matrix.name }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Prepare local source crate patches - shell: bash - run: | - PYTHON=python3 - if ! command -v python3 >/dev/null 2>&1; then - PYTHON=python - fi - "$PYTHON" -m venv .cargo/sync-venv - VENV_PYTHON=.cargo/sync-venv/bin/python - if [ ! -x "$VENV_PYTHON" ]; then - VENV_PYTHON=.cargo/sync-venv/Scripts/python.exe - fi - "$VENV_PYTHON" -m pip install tomlkit requests InquirerPy - "$VENV_PYTHON" scripts/sync-versions.py \ - --prepare-local-patches .cargo/tidesdb-src \ - --write-cargo-patch-config .cargo/config.toml \ - --versions current \ - --yes - - - name: Install dependencies (Linux) - if: runner.os == 'Linux' - run: | - sudo apt update - sudo apt install -y libzstd-dev liblz4-dev libsnappy-dev build-essential cmake pkg-config - - - name: Install dependencies (macOS) - if: runner.os == 'macOS' - run: brew install zstd lz4 snappy cmake - - - name: Setup MSYS2 (Windows) - if: runner.os == 'Windows' - uses: msys2/setup-msys2@v2 - with: - msystem: MINGW64 - update: true - install: >- - mingw-w64-x86_64-gcc - mingw-w64-x86_64-cmake - mingw-w64-x86_64-make - mingw-w64-x86_64-zstd - mingw-w64-x86_64-lz4 - mingw-w64-x86_64-snappy - mingw-w64-x86_64-rust - - - name: Create CMake config files for MSYS2 packages (Windows) - if: runner.os == 'Windows' - shell: msys2 {0} - run: | - MINGW_PREFIX_WIN=$(cygpath -m /mingw64) - - mkdir -p /mingw64/lib/cmake/lz4 - mkdir -p /mingw64/lib/cmake/zstd - mkdir -p /mingw64/lib/cmake/Snappy - mkdir -p /mingw64/lib/cmake/PThreads4W - - cat > /mingw64/lib/cmake/lz4/lz4-config.cmake << EOF - if(NOT TARGET lz4::lz4) - add_library(lz4::lz4 SHARED IMPORTED) - set_target_properties(lz4::lz4 PROPERTIES - IMPORTED_LOCATION "${MINGW_PREFIX_WIN}/bin/liblz4.dll" - IMPORTED_IMPLIB "${MINGW_PREFIX_WIN}/lib/liblz4.dll.a" - INTERFACE_INCLUDE_DIRECTORIES "${MINGW_PREFIX_WIN}/include" - ) - endif() - EOF - - cat > /mingw64/lib/cmake/zstd/zstd-config.cmake << EOF - if(NOT TARGET zstd::libzstd_shared) - add_library(zstd::libzstd_shared SHARED IMPORTED) - set_target_properties(zstd::libzstd_shared PROPERTIES - IMPORTED_LOCATION "${MINGW_PREFIX_WIN}/bin/libzstd.dll" - IMPORTED_IMPLIB "${MINGW_PREFIX_WIN}/lib/libzstd.dll.a" - INTERFACE_INCLUDE_DIRECTORIES "${MINGW_PREFIX_WIN}/include" - ) - endif() - EOF - - cat > /mingw64/lib/cmake/Snappy/Snappy-config.cmake << EOF - if(NOT TARGET Snappy::snappy) - add_library(Snappy::snappy SHARED IMPORTED) - set_target_properties(Snappy::snappy PROPERTIES - IMPORTED_LOCATION "${MINGW_PREFIX_WIN}/bin/libsnappy.dll" - IMPORTED_IMPLIB "${MINGW_PREFIX_WIN}/lib/libsnappy.dll.a" - INTERFACE_INCLUDE_DIRECTORIES "${MINGW_PREFIX_WIN}/include" - ) - endif() - EOF - - cat > /mingw64/lib/cmake/PThreads4W/PThreads4W-config.cmake << EOF - if(NOT TARGET PThreads4W::PThreads4W) - add_library(PThreads4W::PThreads4W SHARED IMPORTED) - set_target_properties(PThreads4W::PThreads4W PROPERTIES - IMPORTED_LOCATION "${MINGW_PREFIX_WIN}/bin/libwinpthread-1.dll" - IMPORTED_IMPLIB "${MINGW_PREFIX_WIN}/lib/libpthread.dll.a" - INTERFACE_INCLUDE_DIRECTORIES "${MINGW_PREFIX_WIN}/include" - ) - endif() - EOF - - - name: Install Rust toolchain - if: runner.os != 'Windows' - uses: dtolnay/rust-toolchain@stable - with: - toolchain: 1.85.0 - - - name: Build from source (Linux) - if: runner.os == 'Linux' - run: cargo build --release - - - name: Build from source (macOS) - if: runner.os == 'macOS' - run: cargo build --release - - - name: Build from source (Windows) - if: runner.os == 'Windows' - shell: msys2 {0} - run: | - export PATH="/mingw64/bin:$PATH" - cargo build --release - - build-from-source-no-compression: - runs-on: ubuntu-latest - name: Linux x64 (no compression) - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Prepare local source crate patches - shell: bash - run: | - PYTHON=python3 - if ! command -v python3 >/dev/null 2>&1; then - PYTHON=python - fi - "$PYTHON" -m venv .cargo/sync-venv - VENV_PYTHON=.cargo/sync-venv/bin/python - if [ ! -x "$VENV_PYTHON" ]; then - VENV_PYTHON=.cargo/sync-venv/Scripts/python.exe - fi - "$VENV_PYTHON" -m pip install tomlkit requests InquirerPy - "$VENV_PYTHON" scripts/sync-versions.py \ - --prepare-local-patches .cargo/tidesdb-src \ - --write-cargo-patch-config .cargo/config.toml \ - --local-patch-archive-versions all \ - --versions current \ - --yes - - - name: Install dependencies - run: | - sudo apt update - sudo apt install -y build-essential cmake pkg-config - - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@stable - with: - toolchain: 1.85.0 - - - name: Build from source without compression - run: cargo build --release --no-default-features --features v9_3_15 - - - name: Build retained source version without compression - run: cargo build --release --no-default-features --features v9_3_6 - - build-from-source-compression-gates: - strategy: - fail-fast: false - matrix: - include: - - name: compression - features: v9_3_15,compression - packages: libzstd-dev liblz4-dev libsnappy-dev - - name: snappy - features: v9_3_15,snappy - packages: libsnappy-dev - - name: lz4 - features: v9_3_15,lz4 - packages: liblz4-dev - - name: zstd - features: v9_3_15,zstd - packages: libzstd-dev - - runs-on: ubuntu-latest - name: Linux x64 (${{ matrix.name }} compression gate) - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Prepare local source crate patches - shell: bash - run: | - PYTHON=python3 - if ! command -v python3 >/dev/null 2>&1; then - PYTHON=python - fi - "$PYTHON" -m venv .cargo/sync-venv - VENV_PYTHON=.cargo/sync-venv/bin/python - if [ ! -x "$VENV_PYTHON" ]; then - VENV_PYTHON=.cargo/sync-venv/Scripts/python.exe - fi - "$VENV_PYTHON" -m pip install tomlkit requests InquirerPy - "$VENV_PYTHON" scripts/sync-versions.py \ - --prepare-local-patches .cargo/tidesdb-src \ - --write-cargo-patch-config .cargo/config.toml \ - --versions current \ - --yes - - - name: Install dependencies - run: | - sudo apt update - sudo apt install -y build-essential cmake pkg-config ${{ matrix.packages }} - - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@stable - with: - toolchain: 1.85.0 - - - name: Build source compression gate - run: cargo build --release --no-default-features --features ${{ matrix.features }} - - build-from-source-objectstore: - strategy: - fail-fast: false - matrix: - include: - - os: ubuntu-latest - name: Linux x64 (objectstore) - - os: ubuntu-24.04-arm - name: Linux ARM64 (objectstore) - - os: macos-latest - name: macOS x64 (objectstore) - - os: windows-latest - name: Windows x64 (objectstore) - - runs-on: ${{ matrix.os }} - name: ${{ matrix.name }} - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Prepare local source crate patches - shell: bash - run: | - PYTHON=python3 - if ! command -v python3 >/dev/null 2>&1; then - PYTHON=python - fi - "$PYTHON" -m venv .cargo/sync-venv - VENV_PYTHON=.cargo/sync-venv/bin/python - if [ ! -x "$VENV_PYTHON" ]; then - VENV_PYTHON=.cargo/sync-venv/Scripts/python.exe - fi - "$VENV_PYTHON" -m pip install tomlkit requests InquirerPy - "$VENV_PYTHON" scripts/sync-versions.py \ - --prepare-local-patches .cargo/tidesdb-src \ - --write-cargo-patch-config .cargo/config.toml \ - --versions current \ - --yes - - - name: Install dependencies (Linux) - if: runner.os == 'Linux' - run: | - sudo apt update - sudo apt install -y libzstd-dev liblz4-dev libsnappy-dev libcurl4-openssl-dev libssl-dev build-essential cmake pkg-config - - - name: Install dependencies (macOS) - if: runner.os == 'macOS' - run: brew install zstd lz4 snappy cmake curl openssl - - - name: Setup MSYS2 (Windows) - if: runner.os == 'Windows' - uses: msys2/setup-msys2@v2 - with: - msystem: MINGW64 - update: true - install: >- - mingw-w64-x86_64-gcc - mingw-w64-x86_64-cmake - mingw-w64-x86_64-make - mingw-w64-x86_64-zstd - mingw-w64-x86_64-lz4 - mingw-w64-x86_64-snappy - mingw-w64-x86_64-curl - mingw-w64-x86_64-openssl - mingw-w64-x86_64-rust - - - name: Create CMake config files for MSYS2 packages (Windows) - if: runner.os == 'Windows' - shell: msys2 {0} - run: | - MINGW_PREFIX_WIN=$(cygpath -m /mingw64) - - mkdir -p /mingw64/lib/cmake/lz4 - mkdir -p /mingw64/lib/cmake/zstd - mkdir -p /mingw64/lib/cmake/Snappy - mkdir -p /mingw64/lib/cmake/PThreads4W - - cat > /mingw64/lib/cmake/lz4/lz4-config.cmake << EOF - if(NOT TARGET lz4::lz4) - add_library(lz4::lz4 SHARED IMPORTED) - set_target_properties(lz4::lz4 PROPERTIES - IMPORTED_LOCATION "${MINGW_PREFIX_WIN}/bin/liblz4.dll" - IMPORTED_IMPLIB "${MINGW_PREFIX_WIN}/lib/liblz4.dll.a" - INTERFACE_INCLUDE_DIRECTORIES "${MINGW_PREFIX_WIN}/include" - ) - endif() - EOF - - cat > /mingw64/lib/cmake/zstd/zstd-config.cmake << EOF - if(NOT TARGET zstd::libzstd_shared) - add_library(zstd::libzstd_shared SHARED IMPORTED) - set_target_properties(zstd::libzstd_shared PROPERTIES - IMPORTED_LOCATION "${MINGW_PREFIX_WIN}/bin/libzstd.dll" - IMPORTED_IMPLIB "${MINGW_PREFIX_WIN}/lib/libzstd.dll.a" - INTERFACE_INCLUDE_DIRECTORIES "${MINGW_PREFIX_WIN}/include" - ) - endif() - EOF - - cat > /mingw64/lib/cmake/Snappy/Snappy-config.cmake << EOF - if(NOT TARGET Snappy::snappy) - add_library(Snappy::snappy SHARED IMPORTED) - set_target_properties(Snappy::snappy PROPERTIES - IMPORTED_LOCATION "${MINGW_PREFIX_WIN}/bin/libsnappy.dll" - IMPORTED_IMPLIB "${MINGW_PREFIX_WIN}/lib/libsnappy.dll.a" - INTERFACE_INCLUDE_DIRECTORIES "${MINGW_PREFIX_WIN}/include" - ) - endif() - EOF - - cat > /mingw64/lib/cmake/PThreads4W/PThreads4W-config.cmake << EOF - if(NOT TARGET PThreads4W::PThreads4W) - add_library(PThreads4W::PThreads4W SHARED IMPORTED) - set_target_properties(PThreads4W::PThreads4W PROPERTIES - IMPORTED_LOCATION "${MINGW_PREFIX_WIN}/bin/libwinpthread-1.dll" - IMPORTED_IMPLIB "${MINGW_PREFIX_WIN}/lib/libpthread.dll.a" - INTERFACE_INCLUDE_DIRECTORIES "${MINGW_PREFIX_WIN}/include" - ) - endif() - EOF - - - name: Install Rust toolchain - if: runner.os != 'Windows' - uses: dtolnay/rust-toolchain@stable - with: - toolchain: 1.85.0 - - - name: Build from source with objectstore (Unix) - if: runner.os != 'Windows' - run: cargo build --release --features objectstore - - - name: Build from source with objectstore (Windows) - if: runner.os == 'Windows' - shell: msys2 {0} - run: | - export PATH="/mingw64/bin:$PATH" - cargo build --release --features objectstore diff --git a/.gitignore b/.gitignore index 2593226..9678cbc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,14 +1,35 @@ -target/ +# Rust build output. Cargo.lock is not committed for a library crate. +/target/ Cargo.lock -__pycache__/ -*.py[cod] -*.swp -*.swo -*~ +**/*.rs.bk +*.profraw +*.profdata + +# Out-of-tree CMake builds of the bundled TidesDB (cargo builds it under target/) +/build/ +/tidesdb-build/ +/cmake-build-*/ + +# Database directories created by running the examples in the README and doc/rust.md +/mydb/ +/mydb-v10/ +/mydb-backup/ +/example_db/ +/testdb/ +*.db + +# Editors and OS files .idea/ .vscode/ *.iml +*.swp +*.swo +*~ .DS_Store Thumbs.db -testdb/ -*.db + +# Patch leftovers and core dumps +*.orig +*.rej +core +core.* diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..0966a5b --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "tidesdb"] + path = tidesdb + url = https://github.com/tidesdb/tidesdb.git diff --git a/Cargo.toml b/Cargo.toml index a8d5239..2ec5d24 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,9 +1,10 @@ [package] name = "tidesdb" -version = "0.11.2" +version = "1.0.0" edition = "2024" +rust-version = "1.85" authors = ["TidesDB "] -description = "TidesDB is a high-performance embeddable, durable, adaptive, and optionally cloud-native key-value storage engine" +description = "TidesDB is a high-performance embeddable, durable, adaptive key-value storage engine" license = "MPL-2.0" repository = "https://github.com/tidesdb/tidesdb-rs" homepage = "https://tidesdb.com" @@ -11,6 +12,23 @@ documentation = "https://tidesdb.com/reference/rust/" readme = "README.md" keywords = ["database", "key-value", "storage", "lsm-tree", "storageengine"] categories = ["database", "data-structures"] +# The C library is linked exactly once per build; two versions of this crate cannot coexist. +links = "tidesdb" +# The bundled TidesDB source (git submodule at tidesdb/, pinned to the release in +# src/ffi.rs TIDESDB_VERSION) ships inside the package; only what the library build needs. +include = [ + "/src/**", + "/build.rs", + "/Cargo.toml", + "/README.md", + "/LICENSE", + "/tidesdb/CMakeLists.txt", + "/tidesdb/LICENSE", + "/tidesdb/cmake/**", + "/tidesdb/include/**", + "/tidesdb/src/**", + "/tidesdb/external/**", +] [dependencies] libc = "0.2" @@ -19,24 +37,23 @@ thiserror = "1.0" [build-dependencies] pkg-config = "0.3" cmake = "0.1" -flate2 = "1" -tar = "0.4" -cfg-if = "1" -tidesdb-src-v9-3-6 = {version = "0.1", optional = true} -tidesdb-src-v9-3-15 = {version = "0.1", optional = true} [dev-dependencies] tempfile = "3.10" [features] -default = ["compression", "v9_3_15"] +# TidesDB needs no compression library; each codec is opt-in and links its system library. +default = [] compression = ["snappy", "lz4", "zstd"] snappy = [] lz4 = [] zstd = [] -objectstore = [] -v9_3_6 = ["dep:tidesdb-src-v9-3-6"] -v9_3_15 = ["dep:tidesdb-src-v9-3-15"] +# Alternative allocators, at most one. Each links the system library, which replaces +# malloc/free for the whole process (TidesDB and Rust alike), as TidesDB's own +# TIDESDB_WITH_*ALLOC CMake options do. +jemalloc = [] +mimalloc = [] +tcmalloc = [] [lib] name = "tidesdb" diff --git a/README.md b/README.md index 02a919c..7fd2c96 100644 --- a/README.md +++ b/README.md @@ -4,19 +4,19 @@ tidesdb-rs is the official Rust binding for TidesDB. TidesDB is a fast and efficient key-value storage engine library written in C. The underlying data structure is based on a log-structured merge-tree (LSM-tree). This Rust binding provides a safe, idiomatic Rust interface to TidesDB with full support for all features. -## Features - -- MVCC with five isolation levels from READ UNCOMMITTED to SERIALIZABLE -- Column families (isolated key-value stores with independent configuration) -- Bidirectional iterators with forward/backward traversal and seek support -- TTL (time to live) support with automatic key expiration -- LZ4, LZ4 Fast, ZSTD, Snappy, or no compression -- Bloom filters with configurable false positive rates -- Global block CLOCK cache for hot blocks -- Savepoints for partial transaction rollback -- Six built-in comparators plus custom registration - -For Rust usage you can go to the TidesDB Rust Reference [here](https://tidesdb.com/reference/rust/). +For the Rust reference see [doc/rust.md](doc/rust.md) or the [TidesDB Rust Reference](https://tidesdb.com/docs/v10/bindings/rust/reference/). + +## Versioning + +The crate's major version maps to a TidesDB major: crate N.x binds TidesDB (N+9).x. + +| Crate | TidesDB | +|-------|---------| +| 0.x | 9.x | +| 1.x | 10.x | +| 2.x | 11.x | + +This release bundles TidesDB 10.2.1. The build ignores a system TidesDB of any other major and says so in a build warning. TidesDB 10 cannot open TidesDB 9 data and there is no migration tool; see [Upgrading from TidesDB 9](doc/rust.md#upgrading-from-tidesdb-9-crate-0x). ## License diff --git a/build.rs b/build.rs index 929f34f..688d2de 100644 --- a/build.rs +++ b/build.rs @@ -3,309 +3,408 @@ // // Licensed under the Mozilla Public License, v. 2.0 (the "License"); -use std::path::PathBuf; +//! Links the TidesDB C library. +//! +//! The crate's major version maps to a TidesDB major: crate N.x binds TidesDB (N+9).x, so +//! 1.x binds TidesDB 10.x and 2.x will bind TidesDB 11.x. A system install found through +//! pkg-config is used when it belongs to that major and is at least the bundled release; +//! anything else is ignored with a warning, and the source bundled in `tidesdb/` (a git +//! submodule pinned to `TIDESDB_VERSION`) is built with CMake and linked statically. -/// Derives the tidesdb version from enabled `v*` Cargo features. -/// Feature `v9_0_5` maps to version `9.0.5`. When multiple version features -/// are enabled (e.g. via dependency unification), the highest is selected. -fn selected_version() -> String { - let mut selected: Option<(u32, u32, u32)> = None; - for (key, _) in std::env::vars() { - if let Some(feature) = key.strip_prefix("CARGO_FEATURE_V") { - let parts: Vec = feature.split('_').filter_map(|s| s.parse().ok()).collect(); - if let [a, b, c] = parts.as_slice() { - let v = (*a, *b, *c); - selected = Some(match selected { - Some(prev) if prev >= v => prev, - _ => v, - }); - } - } - } - match selected { - Some((a, b, c)) => format!("{a}.{b}.{c}"), - None => "9.3.15".to_string(), - } -} - -fn parse_version(version: &str) -> Option<(u32, u32, u32)> { - let parts: Vec = version.split('.').filter_map(|s| s.parse().ok()).collect(); - match parts.as_slice() { - [a, b, c] => Some((*a, *b, *c)), - _ => None, - } -} - -fn version_at_least(version: &str, major: u32, minor: u32, patch: u32) -> bool { - parse_version(version).is_some_and(|v| v >= (major, minor, patch)) -} - -// BEGIN GENERATED TIDESDB SOURCE ARCHIVES -cfg_if::cfg_if! { - if #[cfg(feature = "v9_3_15")] { - fn source_archive() -> PathBuf { - tidesdb_src_v9_3_15::archive_path() - } - } - else if #[cfg(feature = "v9_3_6")] { - fn source_archive() -> PathBuf { - tidesdb_src_v9_3_6::archive_path() - } - } - else { - fn source_archive() -> PathBuf { - panic!("no tidesdb source crate is enabled; enable a vX_Y_Z feature") - } - } -} - -// END GENERATED TIDESDB SOURCE ARCHIVES +use std::path::{Path, PathBuf}; -fn extract_archive(version: &str, out_dir: &str) -> PathBuf { - let archive_path = source_archive(); - let extract_dir = PathBuf::from(out_dir).join("tidesdb-source"); - let src_dir = extract_dir.join(format!("tidesdb-{version}")); +/// The TidesDB release the bindings in src/ffi.rs were written against and the submodule is +/// pinned to. Must match `TIDESDB_VERSION` in src/ffi.rs. +const TIDESDB_VERSION: &str = "10.2.1"; - if src_dir.exists() { - return src_dir; - } +/// Crate major N binds TidesDB major N + this. +const TIDESDB_MAJOR_OFFSET: u64 = 9; - let tarball = std::fs::File::open(&archive_path).unwrap_or_else(|e| { - panic!( - "failed to open TidesDB v{version} source archive {}: {e}", - archive_path.display() - ) - }); - let decoder = flate2::read::GzDecoder::new(tarball); - let mut archive = tar::Archive::new(decoder); - std::fs::create_dir_all(&extract_dir).expect("Failed to create extract directory"); - archive - .unpack(&extract_dir) - .expect("Failed to extract TidesDB source archive"); - - src_dir +fn feature(name: &str) -> bool { + std::env::var(format!("CARGO_FEATURE_{}", name.to_uppercase())).is_ok() } -fn with_objectstore() -> bool { - std::env::var("CARGO_FEATURE_OBJECTSTORE").is_ok() +fn on_off(enabled: bool) -> &'static str { + if enabled { "ON" } else { "OFF" } } -fn with_snappy() -> bool { - std::env::var("CARGO_FEATURE_SNAPPY").is_ok() +/// `major.minor.patch`, ignoring any suffix; missing parts are 0. +fn parse_version(v: &str) -> Option<(u64, u64, u64)> { + let mut parts = v.trim().split(['.', '-', '+']).map(|p| p.parse::()); + let major = parts.next()?.ok()?; + let minor = parts.next().and_then(|p| p.ok()).unwrap_or(0); + let patch = parts.next().and_then(|p| p.ok()).unwrap_or(0); + Some((major, minor, patch)) } -fn with_lz4() -> bool { - std::env::var("CARGO_FEATURE_LZ4").is_ok() +/// The TidesDB major this crate version binds, checked against `TIDESDB_VERSION`. +fn tidesdb_major() -> u64 { + let crate_version = env!("CARGO_PKG_VERSION"); + let crate_major: u64 = env!("CARGO_PKG_VERSION_MAJOR").parse().unwrap(); + let major = crate_major + TIDESDB_MAJOR_OFFSET; + let bound = parse_version(TIDESDB_VERSION).unwrap().0; + assert_eq!( + major, bound, + "tidesdb crate {crate_version} must bind TidesDB {major}.x (crate N.x binds TidesDB \ + (N+{TIDESDB_MAJOR_OFFSET}).x), but build.rs pins TidesDB {TIDESDB_VERSION}. Bump the \ + crate major or the pinned release so they agree." + ); + major } -fn with_zstd() -> bool { - std::env::var("CARGO_FEATURE_ZSTD").is_ok() +fn bundled_source() -> PathBuf { + let dir = PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").unwrap()).join("tidesdb"); + let cmake_lists = dir.join("CMakeLists.txt"); + let Ok(text) = std::fs::read_to_string(&cmake_lists) else { + panic!( + "TidesDB source not found at {}. In a git checkout run \ + `git submodule update --init`, or install TidesDB {TIDESDB_VERSION} (or a later \ + {}.x) with a pkg-config file.", + dir.display(), + parse_version(TIDESDB_VERSION).unwrap().0 + ); + }; + + // project(tidesdb VERSION x.y.z ...) -- the submodule must be checked out at the pinned tag + let found = text + .lines() + .position(|l| l.trim_start().starts_with("project(")) + .and_then(|start| { + let call: String = text.lines().skip(start).collect::>().join(" "); + let call = call.split(')').next()?.to_string(); + let mut words = call.split_whitespace().skip_while(|w| *w != "VERSION"); + words.nth(1).map(str::to_string) + }); + if found.as_deref() != Some(TIDESDB_VERSION) { + panic!( + "the bundled TidesDB source at {} is version {}, but this crate binds TidesDB \ + {TIDESDB_VERSION}. Check out the pinned release: \ + `git -C tidesdb checkout v{TIDESDB_VERSION}`.", + dir.display(), + found.as_deref().unwrap_or("unknown") + ); + } + dir } -fn build_from_source(version: &str) -> PathBuf { - let out_dir = std::env::var("OUT_DIR").unwrap(); - let src_dir = extract_archive(version, &out_dir); - - let mut cfg = cmake::Config::new(&src_dir); +fn build_from_source(src: &Path) -> PathBuf { + let mut cfg = cmake::Config::new(src); cfg.define("TIDESDB_BUILD_TESTS", "OFF") - .define("BUILD_SHARED_LIBS", "OFF"); - - if version_at_least(version, 9, 3, 4) { - cfg.define( - "TIDESDB_WITH_SNAPPY", - if with_snappy() { "ON" } else { "OFF" }, - ) - .define("TIDESDB_WITH_LZ4", if with_lz4() { "ON" } else { "OFF" }) - .define("TIDESDB_WITH_ZSTD", if with_zstd() { "ON" } else { "OFF" }); - } - - if with_objectstore() { - cfg.define("TIDESDB_WITH_S3", "ON"); - } - + .define("TIDESDB_BUILD_BENCH", "OFF") + .define("TIDESDB_BUILD_FUZZERS", "OFF") + .define("TIDESDB_WARN_STRICT", "OFF") + .define("BUILD_SHARED_LIBS", "OFF") + // set explicitly: the C build picks a platform-dependent default for Snappy + .define("TIDESDB_WITH_SNAPPY", on_off(feature("snappy"))) + .define("TIDESDB_WITH_LZ4", on_off(feature("lz4"))) + .define("TIDESDB_WITH_ZSTD", on_off(feature("zstd"))) + // allocators only change what the final binary links (see link_allocator), so the + // static archive is the same either way; keep CMake from probing for them + .define("TIDESDB_WITH_JEMALLOC", "OFF") + .define("TIDESDB_WITH_MIMALLOC", "OFF") + .define("TIDESDB_WITH_TCMALLOC", "OFF"); cfg.build() } -fn brew_prefix() -> Option { - std::process::Command::new("brew") - .arg("--prefix") - .output() - .ok() - .filter(|o| o.status.success()) - .map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string()) -} - -#[cfg(target_os = "windows")] -fn version_less_than(version: &str, major: u32, minor: u32, patch: u32) -> bool { - !version_at_least(version, major, minor, patch) +/// A system library: pkg-config names to try, the name to link when none is found, and the +/// package to suggest for Debian/Ubuntu, Homebrew and vcpkg. +struct SysLib { + pkg: &'static [&'static str], + lib: &'static str, + debian: &'static str, + brew: &'static str, + vcpkg: &'static str, } -fn main() { - let version = selected_version(); - - println!("cargo:rustc-check-cfg=cfg(tidesdb_has_txn_single_delete)"); - println!("cargo:rustc-check-cfg=cfg(tidesdb_has_compact_range)"); - println!("cargo:rustc-check-cfg=cfg(tidesdb_has_max_concurrent_flushes)"); - println!("cargo:rustc-check-cfg=cfg(tidesdb_has_tombstone_stats)"); - println!("cargo:rustc-check-cfg=cfg(tidesdb_has_cancel_background_work)"); - println!("cargo:rustc-check-cfg=cfg(tidesdb_has_raise_open_file_limit)"); - println!("cargo:rustc-check-cfg=cfg(tidesdb_has_write_amp_stats)"); - println!("cargo:rustc-check-cfg=cfg(tidesdb_has_single_writer_fencing)"); - - // These gates follow C API/ABI boundaries found while testing the version - // matrix. Keeping them derived from the selected C tag avoids long feature - // lists and prevents Rust FFI structs from reading fields older tags do not - // expose. - if version_at_least(&version, 9, 1, 0) { - println!("cargo:rustc-cfg=tidesdb_has_txn_single_delete"); - } - if version_at_least(&version, 9, 2, 0) { - println!("cargo:rustc-cfg=tidesdb_has_compact_range"); - println!("cargo:rustc-cfg=tidesdb_has_max_concurrent_flushes"); - println!("cargo:rustc-cfg=tidesdb_has_tombstone_stats"); +/// Link each library through pkg-config, falling back to linking it by name with a warning +/// that lists how to install the ones that were not found. +fn link_system_libs(what: &str, libs: &[SysLib]) { + let missing: Vec<&SysLib> = libs + .iter() + .filter(|l| !l.pkg.iter().any(|p| pkg_config::probe_library(p).is_ok())) + .collect(); + for l in &missing { + println!("cargo:rustc-link-lib={}", l.lib); } - if version_at_least(&version, 9, 3, 2) { - println!("cargo:rustc-cfg=tidesdb_has_cancel_background_work"); + if !missing.is_empty() { + let names: Vec<_> = missing.iter().map(|l| l.lib).collect(); + let join = |f: fn(&SysLib) -> &'static str| { + missing.iter().map(|l| f(l)).collect::>().join(" ") + }; + println!( + "cargo:warning=Could not find {what} {} via pkg-config, falling back to link by \ + name. If linking fails, install them:\ + \n\x20 Debian/Ubuntu: sudo apt install {}\ + \n\x20 macOS: brew install {}\ + \n\x20 Windows: vcpkg install {}", + names.join(", "), + join(|l| l.debian), + join(|l| l.brew), + join(|l| l.vcpkg), + ); } - if version_at_least(&version, 9, 3, 3) { - println!("cargo:rustc-cfg=tidesdb_has_raise_open_file_limit"); +} + +fn link_compression_deps() { + let mut libs = Vec::new(); + if feature("zstd") { + libs.push(SysLib { + pkg: &["libzstd"], + lib: "zstd", + debian: "libzstd-dev", + brew: "zstd", + vcpkg: "zstd:x64-windows", + }); } - if version_at_least(&version, 9, 3, 4) { - println!("cargo:rustc-cfg=tidesdb_has_write_amp_stats"); + if feature("lz4") { + libs.push(SysLib { + pkg: &["liblz4"], + lib: "lz4", + debian: "liblz4-dev", + brew: "lz4", + vcpkg: "lz4:x64-windows", + }); } - if version_at_least(&version, 9, 3, 8) { - println!("cargo:rustc-cfg=tidesdb_has_single_writer_fencing"); + if feature("snappy") { + libs.push(SysLib { + pkg: &["snappy"], + lib: "snappy", + debian: "libsnappy-dev", + brew: "snappy", + vcpkg: "snappy:x64-windows", + }); } + link_system_libs("compression libraries", &libs); +} - // tidesdb S3 connector uses POSIX-only functions (gmtime_r, fmemopen) - // that were not available on Windows until v9.0.6 - #[cfg(target_os = "windows")] - if with_objectstore() && version_less_than(&version, 9, 0, 6) { +/// Link the alternative allocator selected by a feature. Like TidesDB's own +/// `TIDESDB_WITH_*ALLOC` CMake options, this only links the allocator: it replaces +/// `malloc`/`free` for the whole process, the Rust side included. +fn link_allocator() { + let selected: Vec<&str> = ["jemalloc", "mimalloc", "tcmalloc"] + .into_iter() + .filter(|f| feature(f)) + .collect(); + if selected.len() > 1 { panic!( - "The `objectstore` feature requires tidesdb >= 9.0.6 on Windows. \ - Versions before 9.0.6 use POSIX-only functions (gmtime_r, fmemopen)." + "the tidesdb features {} each replace the process allocator; enable at most one", + selected.join(", ") ); } - - // Try pkg-config with exact version match - if pkg_config::Config::new() - .exactly_version(&version) - .probe("tidesdb") - .is_ok() - { + let Some(&alloc) = selected.first() else { return; + }; + let windows = std::env::var("CARGO_CFG_TARGET_OS").as_deref() == Ok("windows"); + if windows && alloc != "mimalloc" { + panic!("the tidesdb `{alloc}` feature is not supported on Windows; use `mimalloc`"); } + let lib = match alloc { + "jemalloc" => SysLib { + pkg: &["jemalloc"], + lib: "jemalloc", + debian: "libjemalloc-dev", + brew: "jemalloc", + vcpkg: "jemalloc:x64-windows", + }, + "mimalloc" => SysLib { + pkg: &["mimalloc"], + lib: "mimalloc", + debian: "libmimalloc-dev", + brew: "mimalloc", + vcpkg: "mimalloc:x64-windows", + }, + _ => SysLib { + pkg: &["libtcmalloc", "libtcmalloc_minimal"], + lib: "tcmalloc", + debian: "libgoogle-perftools-dev", + brew: "gperftools", + vcpkg: "gperftools:x64-windows", + }, + }; + link_system_libs("allocator", &[lib]); +} - // Build from source - let dst = build_from_source(&version); - - println!("cargo:rustc-link-search=native={}/lib", dst.display()); - println!("cargo:rustc-link-lib=static=tidesdb"); - - let mut missing = Vec::new(); - let compression_deps = [ - ("libzstd", "zstd", with_zstd()), - ("liblz4", "lz4", with_lz4()), - ("snappy", "snappy", with_snappy()), - ]; - let always_needs_compression_deps = !version_at_least(&version, 9, 3, 4); - - if always_needs_compression_deps || compression_deps.iter().any(|(_, _, enabled)| *enabled) { - // Older TidesDB releases always compile compression backends into the C - // library. v9.3.4+ can gate each optional codec independently. - for (dep, lib_name, enabled) in compression_deps { - if !always_needs_compression_deps && !enabled { - continue; - } - if pkg_config::probe_library(dep).is_err() { - println!("cargo:rustc-link-lib={lib_name}"); - missing.push(lib_name); - } - } +/// Use a TidesDB found through pkg-config if it belongs to the bound major and is at least the +/// bundled release. Returns the version linked, or `None` to build the bundled source. +fn system_tidesdb(major: u64) -> Option { + if std::env::var_os("TIDESDB_NO_PKG_CONFIG").is_some() { + return None; } - // Link S3/object store dependencies (libcurl + openssl) - if with_objectstore() { - if pkg_config::probe_library("libcurl").is_err() { - println!("cargo:rustc-link-lib=curl"); - missing.push("curl"); - } - if pkg_config::probe_library("openssl").is_err() { - println!("cargo:rustc-link-lib=ssl"); - println!("cargo:rustc-link-lib=crypto"); - missing.push("ssl/crypto"); - } + // link metadata only once the version is accepted, but rerun when PKG_CONFIG_* changes + let found = pkg_config::Config::new() + .cargo_metadata(false) + .env_metadata(true) + .probe("tidesdb") + .ok()?; + let crate_version = env!("CARGO_PKG_VERSION"); + let crate_major = major - TIDESDB_MAJOR_OFFSET; + let usable = match parse_version(&found.version) { + Some(v) => v.0 == major && v >= parse_version(TIDESDB_VERSION).unwrap(), + None => false, + }; + if !usable { + println!( + "cargo:warning=Ignoring the system TidesDB {} found via pkg-config: tidesdb crate \ + {crate_version} binds TidesDB {major}.x (at least {TIDESDB_VERSION}). Crate \ + {crate_major}.x maps to TidesDB {major}.x, crate {}.x to TidesDB {}.x. Building the \ + bundled TidesDB {TIDESDB_VERSION} instead.", + found.version, + crate_major + 1, + major + 1, + ); + return None; } - if !missing.is_empty() { - let mut msg = format!( - "cargo:warning=Could not find {} via pkg-config, falling back to link by name. \ - If linking fails, install them:", - missing.join(", ") + // probe again, this time emitting the link metadata + let lib = pkg_config::Config::new() + .range_version(found.version.as_str()..=found.version.as_str()) + .probe("tidesdb") + .ok()?; + if lib.version != TIDESDB_VERSION { + println!( + "cargo:warning=Linking the system TidesDB {} found via pkg-config. The bindings \ + were written against TidesDB {TIDESDB_VERSION}; set TIDESDB_NO_PKG_CONFIG=1 to \ + build the bundled {TIDESDB_VERSION} instead.", + lib.version ); - let needs_compression_deps = - always_needs_compression_deps || with_snappy() || with_lz4() || with_zstd(); + } + if feature("snappy") || feature("lz4") || feature("zstd") { + println!( + "cargo:warning=The tidesdb compression features do not change a system TidesDB: it \ + has whichever codecs it was built with. Check CompressionAlgorithm::is_available." + ); + } + Some(lib.version) +} - let mut debian_packages = Vec::new(); - let mut macos_packages = Vec::new(); - let mut windows_packages = Vec::new(); +/// The platform the crate is being built for, from Cargo's target cfg. A build script's own +/// `#[cfg(...)]` describes the machine running the build, which differs when cross-compiling. +struct Target { + os: String, + family: String, + env: String, + pointer_width: String, + /// Whether the build runs on the target platform itself (not cross-compiling) + native: bool, +} - if always_needs_compression_deps || with_zstd() { - debian_packages.push("libzstd-dev"); - macos_packages.push("zstd"); - windows_packages.push("zstd:x64-windows"); +impl Target { + fn from_env() -> Self { + let var = |k: &str| std::env::var(k).unwrap_or_default(); + Target { + os: var("CARGO_CFG_TARGET_OS"), + family: var("CARGO_CFG_TARGET_FAMILY"), + env: var("CARGO_CFG_TARGET_ENV"), + pointer_width: var("CARGO_CFG_TARGET_POINTER_WIDTH"), + native: var("HOST") == var("TARGET"), } - if always_needs_compression_deps || with_lz4() { - debian_packages.push("liblz4-dev"); - macos_packages.push("lz4"); - windows_packages.push("lz4:x64-windows"); + } +} + +/// The system libraries the static TidesDB needs, following the per-platform link recipe in +/// TidesDB's CMakeLists.txt. +fn link_platform_libs(target: &Target) { + let link = |lib: &str| println!("cargo:rustc-link-lib={lib}"); + match target.os.as_str() { + // MSVC uses the native Win32 threads in compat.h; MinGW links winpthreads + "windows" => { + if target.env == "gnu" { + link("pthread"); + } } - if always_needs_compression_deps || with_snappy() { - debian_packages.push("libsnappy-dev"); - macos_packages.push("snappy"); - windows_packages.push("snappy:x64-windows"); + // Bionic has pthreads and libm in libc and ships no libpthread + "android" => link("m"), + "macos" | "ios" | "freebsd" | "openbsd" | "netbsd" | "dragonfly" | "solaris" + | "illumos" => { + link("m"); + link("pthread"); } - if with_objectstore() { - debian_packages.extend(["libcurl4-openssl-dev", "libssl-dev"]); - macos_packages.extend(["curl", "openssl"]); - windows_packages.extend(["curl:x64-windows", "openssl:x64-windows"]); + _ if target.family == "unix" => { + link("pthread"); + link("m"); + // 64-bit atomics on 32-bit targets (ARMv7, PowerPC, MIPS, x86) live in libatomic + if target.pointer_width == "32" { + link("atomic"); + } + // Snappy is C++; its C++ runtime comes along when Snappy is enabled + if feature("snappy") { + link("stdc++"); + } } + _ => {} + } +} - if needs_compression_deps || with_objectstore() { - msg.push_str(&format!( - "\n\x20 Debian/Ubuntu: sudo apt install {}", - debian_packages.join(" ") - )); - msg.push_str(&format!( - "\n\x20 macOS: brew install {}", - macos_packages.join(" ") - )); - msg.push_str(&format!( - "\n\x20 Windows: vcpkg install {}", - windows_packages.join(" ") - )); +/// Library directories of the build machine's package managers, where the codec and +/// allocator libraries usually live. Only added for native builds: when cross-compiling +/// they hold libraries for the wrong platform, and the target's libraries are found through +/// pkg-config (`PKG_CONFIG_SYSROOT_DIR`, `PKG_CONFIG_ALLOW_CROSS`) or `RUSTFLAGS="-L ..."`. +fn link_host_search_paths(target: &Target) { + if !target.native { + return; + } + let search = |dir: &str| println!("cargo:rustc-link-search=native={dir}"); + match target.os.as_str() { + "macos" => match brew_prefix() { + Some(prefix) => search(&format!("{prefix}/lib")), + None => { + search("/opt/homebrew/lib"); + search("/usr/local/lib"); + } + }, + "windows" => { + // MSYS2 shells set MINGW_PREFIX as a POSIX path (/mingw64); the native linker + // needs it under the default MSYS2 root + if target.env == "gnu" { + let prefix = std::env::var("MINGW_PREFIX").unwrap_or_else(|_| "/mingw64".into()); + match prefix.strip_prefix('/') { + Some(rest) => search(&format!("C:/msys64/{rest}/lib")), + None => search(&format!("{prefix}/lib")), + } + } } - println!("{msg}"); + "netbsd" => search("/usr/pkg/lib"), + "solaris" | "illumos" => search("/opt/ooce/lib/amd64"), + _ => search("/usr/local/lib"), } +} - // Platform-specific dependencies - #[cfg(unix)] - { - println!("cargo:rustc-link-lib=pthread"); - } +fn brew_prefix() -> Option { + std::process::Command::new("brew") + .arg("--prefix") + .output() + .ok() + .filter(|o| o.status.success()) + .map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string()) +} - // Add library search paths - if let Some(prefix) = brew_prefix() { - println!("cargo:rustc-link-search=native={prefix}/lib"); - } else { - println!("cargo:rustc-link-search=native=/usr/local/lib"); - println!("cargo:rustc-link-search=native=/usr/lib"); - #[cfg(target_os = "macos")] - { - println!("cargo:rustc-link-search=native=/opt/homebrew/lib"); - } - #[cfg(target_os = "windows")] - { - println!("cargo:rustc-link-search=native=C:/msys64/mingw64/lib"); - } +fn main() { + println!("cargo:rerun-if-changed=build.rs"); + println!("cargo:rerun-if-changed=tidesdb/include/db.h"); + println!("cargo:rerun-if-changed=tidesdb/CMakeLists.txt"); + println!("cargo:rerun-if-env-changed=TIDESDB_NO_PKG_CONFIG"); + println!("cargo:rerun-if-env-changed=MINGW_PREFIX"); + println!("cargo:rustc-check-cfg=cfg(tidesdb_system)"); + + let major = tidesdb_major(); + let target = Target::from_env(); + + if let Some(version) = system_tidesdb(major) { + println!("cargo:rustc-cfg=tidesdb_system"); + println!("cargo:rustc-env=TIDESDB_LINKED_VERSION={version}"); + link_allocator(); + link_host_search_paths(&target); + return; } + + let dst = build_from_source(&bundled_source()); + println!("cargo:rustc-env=TIDESDB_LINKED_VERSION={TIDESDB_VERSION}"); + println!("cargo:rustc-link-search=native={}/lib", dst.display()); + println!("cargo:rustc-link-search=native={}/lib64", dst.display()); + println!("cargo:rustc-link-lib=static=tidesdb"); + + link_compression_deps(); + link_allocator(); + link_platform_libs(&target); + link_host_search_paths(&target); } diff --git a/doc/manual.json b/doc/manual.json new file mode 100644 index 0000000..a611abb --- /dev/null +++ b/doc/manual.json @@ -0,0 +1,19 @@ +{ + "$comment": "Table of contents for the TidesDB Rust binding docs, read by the TidesDB website (rendered under /docs//bindings//). The binding version comes from this repo's release and is pinned per TidesDB major on the site — do not repeat it here. This currently points at the single reference page. To expand it like the core/TideSQL manuals, split the page into numbered part folders (e.g. 01-getting-started/, 02-guide/, 03-reference/) and list each part and chapter below; give each markdown file only 'title' and 'description' frontmatter — the website injects the slug.", + "title": "TidesDB Rust", + "tidesdb": "10.0.0", + "parts": [ + { + "id": "reference", + "title": "API Reference", + "dir": "", + "chapters": [ + { + "file": "rust.md", + "slug": "reference", + "title": "Rust API Reference" + } + ] + } + ] +} diff --git a/doc/rust.md b/doc/rust.md new file mode 100644 index 0000000..b0821c4 --- /dev/null +++ b/doc/rust.md @@ -0,0 +1,1596 @@ +--- +title: TidesDB Rust API Reference +description: Rust API reference for TidesDB +--- + +
+ +If you want to download the source of this document, you can find it [here](https://github.com/tidesdb/tidesdb-rs/blob/master/doc/rust.md). + +
+ +
+ +## Getting Started + +The `tidesdb` crate provides safe Rust bindings for TidesDB, an embeddable key-value storage engine based on a log-structured merge-tree. This release (crate 1.0.0) binds and bundles **TidesDB 10.2.1**. + +### Versioning + +The crate's major version maps to a TidesDB major version: crate **N.x binds TidesDB (N+9).x**. + +| Crate | TidesDB | +|-------|---------| +| 0.x | 9.x | +| 1.x | 10.x | +| 2.x | 11.x (when released) | + +Each TidesDB major gets a new crate major; crate minor and patch releases follow changes to the bindings and to the bundled TidesDB release within that major. Pick the crate major that matches the TidesDB major you want: + +```toml +[dependencies] +tidesdb = "1" # TidesDB 10.x +``` + +Two constants report what you got: + +```rust +// the release the bindings were written against and the bundled source is pinned to +assert_eq!(tidesdb::TIDESDB_VERSION, "10.2.1"); +// the release actually linked: the bundled one, or a system install (see below) +println!("linked TidesDB {}", tidesdb::TIDESDB_LINKED_VERSION); +``` + +The build script enforces the mapping. A system TidesDB of another major (or older than 10.2.1) is never linked: the build prints a warning saying so and builds the bundled release instead, for example: + +```text +warning: tidesdb@1.0.0: Ignoring the system TidesDB 11.0.0 found via pkg-config: tidesdb crate 1.0.0 +binds TidesDB 10.x (at least 10.2.1). Crate 1.x maps to TidesDB 10.x, crate 2.x to TidesDB 11.x. +Building the bundled TidesDB 10.2.1 instead. +``` + +### Prerequisites + +The crate needs Rust 1.85 or newer (it uses the 2024 edition; `rust-version` in its `Cargo.toml` says so). The TidesDB C source is bundled with the crate, so beyond Rust you only need a C compiler and CMake. No system libraries are required by default. + +**Debian/Ubuntu** +```bash +sudo apt install build-essential cmake +``` + +**macOS** +```bash +xcode-select --install +brew install cmake +``` + +**Windows** + +Install Visual Studio Build Tools (or another C toolchain) and CMake. + +### Installing from crates.io + +```toml +[dependencies] +tidesdb = "1" +``` + +Or using `cargo add`: + +```bash +cargo add tidesdb +``` + +### Compression Features + +TidesDB needs no compression library, and the default build has none. Each codec is opt-in through a Cargo feature, and each one links its system library: + +| Feature | Enables | System library | +|---------|---------|----------------| +| `snappy` | `CompressionAlgorithm::Snappy` | snappy | +| `lz4` | `CompressionAlgorithm::Lz4`, `CompressionAlgorithm::Lz4Fast` | liblz4 | +| `zstd` | `CompressionAlgorithm::Zstd` | libzstd | +| `compression` | all three of the above | all three | + +```toml +[dependencies] +tidesdb = { version = "1", features = ["lz4", "zstd"] } +``` + +Install the libraries for the features you enable: + +- Debian/Ubuntu · `sudo apt install libsnappy-dev liblz4-dev libzstd-dev` +- macOS · `brew install snappy lz4 zstd` +- Windows · `vcpkg install snappy:x64-windows lz4:x64-windows zstd:x64-windows` + +The build script looks for each enabled library with `pkg-config` and falls back to linking it by name, printing a warning with the install commands above if it was not found. + +The features choose the codecs of the bundled build. A system TidesDB linked through pkg-config has whichever codecs it was built with, whatever the features say (the build prints a warning if you enable them anyway). Either way, `CompressionAlgorithm::is_available` tells you at runtime what the linked library supports (see [Compression](#compression)). + +### Allocator Features + +TidesDB can run on an alternative memory allocator, matching its `TIDESDB_WITH_JEMALLOC`, `TIDESDB_WITH_MIMALLOC` and `TIDESDB_WITH_TCMALLOC` CMake options. Enable at most one; enabling two fails the build. + +| Feature | Allocator | Debian/Ubuntu | macOS | Windows | +|---------|-----------|---------------|-------|---------| +| `jemalloc` | jemalloc | `libjemalloc-dev` | `brew install jemalloc` | not supported | +| `mimalloc` | mimalloc | `libmimalloc-dev` | `brew install mimalloc` | `vcpkg install mimalloc:x64-windows` | +| `tcmalloc` | tcmalloc (gperftools) | `libgoogle-perftools-dev` | `brew install gperftools` | not supported | + +```toml +[dependencies] +tidesdb = { version = "1", features = ["jemalloc"] } +``` + +The allocator is linked into your binary as a shared library and replaces `malloc` and `free` for the **whole process**: TidesDB, Rust's default (system) allocator, and any other C code. That is how TidesDB's own CMake options work too; TidesDB has no allocator-specific code. If you only want Rust allocations on jemalloc or mimalloc, use a Rust allocator crate with `#[global_allocator]` instead; it does not affect TidesDB. The library is found with `pkg-config` (for tcmalloc, `libtcmalloc` then `libtcmalloc_minimal`), falling back to linking it by name with a warning. The allocator feature works with both the bundled build and a system TidesDB. + +### How the Build System Works + +The build script (`build.rs`) links one TidesDB 10.x release: + +1. **Version mapping check** · The crate major and the pinned TidesDB release must agree (crate 1.x, TidesDB 10.x), and the bundled source must be the pinned release (10.2.1); otherwise the build stops with an error naming the mismatch. +2. **pkg-config check** · If `pkg-config` finds a TidesDB installed on the system that is **10.x and at least 10.2.1**, it is linked and nothing is built. A newer 10.x is linked with a warning that the bindings were written against 10.2.1. Any other version is ignored with the warning shown under [Versioning](#versioning). TidesDB's own CMake install does not write a `tidesdb.pc` file, so this applies only when a package (or you) provides one. Set the `TIDESDB_NO_PKG_CONFIG` environment variable to skip this check and always build the bundled source. +3. **Bundled source** · Otherwise the TidesDB source in the crate's `tidesdb/` directory (a git submodule pinned to the 10.2.1 release, shipped inside the published crate) is built with CMake as a static library and linked into your binary. The `snappy`, `lz4` and `zstd` features are passed to the CMake build explicitly. + +```bash +# Always build the bundled source, even if a matching TidesDB is installed +TIDESDB_NO_PKG_CONFIG=1 cargo build +``` + +The crate declares `links = "tidesdb"`, so only one version of it can be linked into a build. + +### Building from GitHub + +The TidesDB source is a git submodule, so a checkout needs it initialised before building: + +```bash +git clone https://github.com/tidesdb/tidesdb-rs.git +cd tidesdb-rs +git submodule update --init +cargo build --release +cargo test +``` + +If the submodule is missing, the build fails with a message telling you to run `git submodule update --init` (or to install TidesDB 10.x with a pkg-config file). If the submodule is checked out at a different release, the build fails and tells you to check out `v10.2.1`. + +**Using directly from GitHub in your Cargo.toml** + +```toml +[dependencies] +tidesdb = { git = "https://github.com/tidesdb/tidesdb-rs.git" } +``` + +**Using an installed TidesDB through pkg-config** + +TidesDB's `cmake --install` installs the library and headers but no pkg-config file. If your package does not ship one, write a minimal `tidesdb.pc` next to the library: + +```text +prefix=/custom/path +libdir=${prefix}/lib +includedir=${prefix}/include + +Name: tidesdb +Description: TidesDB storage engine +Version: 10.2.1 +Libs: -L${libdir} -ltidesdb -lpthread +Cflags: -I${includedir} +``` + +```bash +# The installed library must be TidesDB 10.x, at least 10.2.1 +export PKG_CONFIG_PATH="/custom/path/lib/pkgconfig:$PKG_CONFIG_PATH" +cargo build +``` + +A TidesDB built with compression links its codec libraries too; add them to `Libs` (for example `-lzstd -llz4 -lsnappy`). When the library is a shared one outside the loader's search path, set `LD_LIBRARY_PATH` (or `DYLD_LIBRARY_PATH` on macOS) when running. + +### Cross-Compiling and Embedded Targets + +The bundled TidesDB is built for the Cargo target, not the build machine, so cross-compiling works like any other C-backed crate. Nothing is downloaded at build time: the C source ships inside the crate. + +TidesDB needs an operating system with a filesystem and threads. Embedded Linux on ARM, AArch64, RISC-V, MIPS or PowerPC is supported; bare-metal (`no_std`) targets are not. + +Point the build at your cross toolchain the usual way; the `cmake` crate passes the C compiler on to TidesDB's CMake build: + +```bash +rustup target add armv7-unknown-linux-gnueabihf +export CC_armv7_unknown_linux_gnueabihf=arm-linux-gnueabihf-gcc +export CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_LINKER=arm-linux-gnueabihf-gcc +cargo build --release --target armv7-unknown-linux-gnueabihf +``` + +A CMake toolchain file (for Yocto, Buildroot or a vendor SDK) can be given with `CMAKE_TOOLCHAIN_FILE`, or per target with `CMAKE_TOOLCHAIN_FILE_`. + +The build script links the system libraries the static TidesDB needs for the **target**, following TidesDB's own CMake recipe: + +| Target | Linked | +|--------|--------| +| Linux and other Unix | `pthread`, `m`; `atomic` on 32-bit targets (ARMv7, PowerPC, MIPS, x86); `stdc++` with the `snappy` feature | +| macOS, iOS, FreeBSD, OpenBSD, NetBSD, DragonFly, Solaris, illumos | `m`, `pthread` | +| Android | `m` (Bionic has pthreads in libc) | +| Windows, GNU (MinGW) | `pthread` (winpthreads) | +| Windows, MSVC | nothing (native Win32 threads) | + +A 32-bit target's toolchain must provide `libatomic` (GCC cross toolchains do). + +Package-manager library directories (Homebrew, `/usr/local/lib`, MSYS2, pkgsrc) are added to the link search path only for native builds. When cross-compiling, the compression and allocator libraries must be built for the target. Make them visible through pkg-config, which the build uses only when cross-compiling is allowed (`PKG_CONFIG_ALLOW_CROSS=1`, with `PKG_CONFIG_SYSROOT_DIR` and `PKG_CONFIG_PATH` pointing into the target sysroot), or with `RUSTFLAGS="-L /path/to/target/libs"`. The same applies to a system TidesDB: when cross-compiling, pkg-config only finds one if it is configured for the target, and otherwise the bundled source is built. + +The CI cross-builds and tests `armv7-unknown-linux-gnueabihf` and `aarch64-unknown-linux-gnu` under QEMU, and cross-builds `x86_64-pc-windows-gnu` from Linux. + +## Upgrading from TidesDB 9 (crate 0.x) + +TidesDB 10 is a new major version. It cannot open a database written by TidesDB 9, and there is no migration tool: to keep your data, read it out with a program built against crate 0.x (TidesDB 9) and write it into a new database with a program built against crate 1.x (TidesDB 10). + +Both crates link a C library exporting the same `tidesdb_*` symbols, so they cannot be linked into one binary. Use two programs, for example one that exports every key and value to a file and one that imports that file: + +```rust +// Importer, built against tidesdb = "1" +use tidesdb::{TidesDB, Config, ColumnFamilyConfig}; + +fn import(pairs: impl IntoIterator, Vec)>) -> tidesdb::Result<()> { + let db = TidesDB::open(Config::new("./mydb-v10"))?; + db.create_column_family("my_cf", ColumnFamilyConfig::default())?; + let cf = db.get_column_family("my_cf")?; + + let mut txn = db.begin_transaction()?; + for (key, value) in pairs { + txn.put(&cf, &key, &value)?; + } + txn.commit()?; + Ok(()) +} +``` + +For large data sets, commit in batches rather than in one transaction. Crate 0.x cannot read a key's expiry back, so keys written with a TTL come across as permanent unless your application knows their lifetimes and writes them with `put_with_ttl`. + +The API changed with the engine as well; this reference describes the 1.x API in full. TidesDB 10 itself no longer has an object store (S3), replica mode, custom comparators or INI configuration files, so the crate has none either. + +## Usage + +### Opening and Closing a Database + +```rust +use tidesdb::{TidesDB, Config, LogLevel, SyncMode}; + +fn main() -> tidesdb::Result<()> { + let config = Config::new("./mydb") + .num_flush_threads(2) + .num_compaction_threads(2) + .log_level(LogLevel::Warn) + .block_cache_size(64 * 1024 * 1024) + .max_open_sstables(256) + .memtable_write_buffer_size(64 * 1024 * 1024) + .memtable_sync_mode(SyncMode::Interval) + .memtable_sync_interval_us(100_000); + + let db = TidesDB::open(config)?; + println!("Database opened"); + + // The database closes when `db` is dropped. + Ok(()) +} +``` + +`Config::new(path)` fills every other field from the C library's defaults (`tidesdb_default_config()`), so the binding tracks the engine's defaults automatically. See the [Database Configuration Reference](#database-configuration-reference) for every field. + +A database directory can be open in only one handle at a time. Opening it again, from this process or another, fails with `ErrorCode::Locked`: + +```rust +use tidesdb::{TidesDB, Config, ErrorCode}; + +let db = TidesDB::open(Config::new("./mydb"))?; +match TidesDB::open(Config::new("./mydb")) { + Err(e) if e.code() == Some(ErrorCode::Locked) => println!("already open"), + Err(e) => return Err(e), + Ok(_) => unreachable!(), +} +``` + +There is no explicit `close`: drop the `TidesDB`. Every `ColumnFamily`, `Transaction`, `Snapshot` and `Iterator` borrows the database, so the compiler guarantees they are all gone before it closes. + +### Column Families + +Column families are isolated key spaces with their own configuration. They share the database's memtable, write-ahead log, value log and block cache. + +#### Creating and Getting + +```rust +use tidesdb::{TidesDB, Config, ColumnFamilyConfig, IsolationLevel}; + +fn main() -> tidesdb::Result<()> { + let db = TidesDB::open(Config::new("./mydb"))?; + + db.create_column_family("users", ColumnFamilyConfig::default())?; + + let config = ColumnFamilyConfig::new() + .level_size_ratio(10) + .enable_bloom_filter(true) + .bloom_fpr(0.01) + .default_isolation_level(IsolationLevel::Snapshot); + db.create_column_family("orders", config)?; + + let users = db.get_column_family("users")?; + println!("handle for {}", users.name()); + + Ok(()) +} +``` + +`create_column_family` returns `ErrorCode::Exists` if the name is taken. `get_column_family` returns `ErrorCode::NotFound` if it does not exist. + +`ColumnFamily` handles are cheap to clone, `Send + Sync`, and borrow the `TidesDB` they came from. Using a handle with a transaction from a different database is rejected with `ErrorCode::InvalidArgs`. + +#### Listing + +```rust +for name in db.list_column_families()? { + println!("{name}"); +} +``` + +#### Renaming + +```rust +db.rename_column_family("old_name", "new_name")?; +``` + +The family is renamed in place, so existing handles stay valid and keep referring to it, but `ColumnFamily::name()` keeps reporting the name the handle was looked up by. `cf.get_stats()?.name` reports the current name. + +#### Cloning + +```rust +db.clone_column_family("users", "users_copy")?; +let copy = db.get_column_family("users_copy")?; +``` + +The clone is a point-in-time copy: everything written to the source before the call is included; later writes to the source are not. Cloning a missing source returns `NotFound`; cloning onto an existing name returns `Exists`. The clone flushes the memtable first and returns `Locked` if the flush queue does not drain in time; retry it. A commit hook on the source is not copied. + +#### Dropping + +Dropping a column family deletes all of its data and cannot be undone. `drop_column_family` takes `&mut self`, so every handle to the family must be gone first: + +```rust +use tidesdb::{TidesDB, Config, ColumnFamilyConfig}; + +fn main() -> tidesdb::Result<()> { + let mut db = TidesDB::open(Config::new("./mydb"))?; + db.create_column_family("scratch", ColumnFamilyConfig::default())?; + { + let cf = db.get_column_family("scratch")?; + // ... use cf ... + } // cf dropped here + + db.drop_column_family("scratch")?; + Ok(()) +} +``` + +When the database is shared (for example behind an `Arc` or across `std::thread::scope` threads) you cannot get `&mut TidesDB`. Use the unchecked variant and uphold its contract yourself: + +```rust +// SAFETY: no ColumnFamily handle for "scratch", and no transaction or iterator +// that has used one, is used after this call. +unsafe { db.drop_column_family_unchecked("scratch")? }; +``` + +### Reading and Writing + +All reads and writes go through a transaction. Writes are buffered until `commit`; dropping an uncommitted transaction rolls it back. `put`, `delete` and the other write methods take `&self`; `commit`, `rollback`, `reset`, `set_timeout` and the two-phase-commit methods take `&mut self`. + +#### Writing Data + +```rust +use tidesdb::{TidesDB, Config, ColumnFamilyConfig}; + +fn main() -> tidesdb::Result<()> { + let db = TidesDB::open(Config::new("./mydb"))?; + db.create_column_family("my_cf", ColumnFamilyConfig::default())?; + let cf = db.get_column_family("my_cf")?; + + let mut txn = db.begin_transaction()?; + txn.put(&cf, b"key", b"value")?; + txn.put(&cf, b"empty", b"")?; // empty values are allowed + txn.commit()?; + + Ok(()) +} +``` + +`put` writes a key that never expires. Keys must be non-empty (an empty key is rejected with `InvalidArgs`). An empty value is allowed and is distinct from an absent key: reads return it as an empty `Vec`. + +#### Writing with TTL + +```rust +use std::time::Duration; + +let mut txn = db.begin_transaction()?; +txn.put_with_ttl(&cf, b"session:abc", b"token", Duration::from_secs(60))?; +txn.commit()?; +``` + +The lifetime is relative. It is rounded up to whole seconds and converted to an absolute deadline when the write is buffered (so `Duration::from_millis(1)` lives about one second). A zero `Duration` is rejected with `ErrorCode::InvalidArgs`. Once the deadline passes, the committed key reads as absent. (A transaction that outlives the deadline of its own uncommitted write still reads that write back with `get`; `get_expiry` reports it absent.) + +#### Reading Data + +```rust +let txn = db.begin_transaction()?; + +let value = txn.get(&cf, b"key")?; +println!("{}", String::from_utf8_lossy(&value)); + +match txn.get(&cf, b"missing") { + Ok(v) => println!("found {v:?}"), + Err(e) if e.is_not_found() => println!("not found"), + Err(e) => return Err(e), +} +``` + +`get` returns an error with `ErrorCode::NotFound` when the key is absent, deleted or expired. A transaction reads its own uncommitted writes, including its own deletes and range deletes. + +| Method | Returns | Conflict tracking | +|--------|---------|-------------------| +| `get(&cf, key)` | `Result>` | Records the read for conflict detection | +| `get_notrack(&cf, key)` | `Result>` | Does not record the read | +| `contains(&cf, key)` | `Result` (`Ok(false)` when absent) | Does not record the read | +| `get_expiry(&cf, key)` | `Result>`: absolute expiry in Unix seconds; `None` = never expires; `NotFound` if the key is absent | Records the read for conflict detection | + +Reads are only recorded at `RepeatableRead` and `Serializable`; at weaker levels nothing is tracked. + +`get_notrack` and `contains` are meant for existence probes, such as uniqueness checks, that should not add to the transaction's read set (and so cannot cause a conflict at `RepeatableRead` or `Serializable`). + +```rust +let txn = db.begin_transaction()?; +if !txn.contains(&cf, b"user:42")? { + println!("free"); +} +if let Some(expires_at) = txn.get_expiry(&cf, b"session:abc")? { + println!("expires at unix time {expires_at}"); +} +``` + +#### Deleting Data + +```rust +let mut txn = db.begin_transaction()?; +txn.delete(&cf, b"key")?; +txn.commit()?; +``` + +#### Single-Delete + +`single_delete` writes a tombstone that reads the same as `delete`, but lets compaction drop the put and the tombstone together: + +```rust +let mut txn = db.begin_transaction()?; +txn.single_delete(&cf, b"once")?; +txn.commit()?; +``` + +:::caution[Contract] +You promise the key was put **at most once** since its previous single-delete (or since it was created). The engine does not verify this; breaking the contract can leave older puts visible. When in doubt, use `delete`. +::: + +#### Range and Prefix Deletes + +```rust +let mut txn = db.begin_transaction()?; + +// every key in [b"b", b"d") +txn.delete_range(&cf, b"b", Some(b"d"))?; + +// every key from b"x" to the end of the family +txn.delete_range(&cf, b"x", None)?; + +// every key beginning with b"user:" +txn.delete_prefix(&cf, b"user:")?; + +txn.commit()?; +``` + +A range or prefix delete costs one entry however many keys it covers. It covers keys written before it, not merely the keys present when it is called, and a newer write inside the range survives it. The lower bound (or prefix) must be non-empty, each bound is at most 256 bytes, and an upper bound must be non-empty and greater than the lower bound; otherwise the call fails with `InvalidArgs`. Use `None` for no upper bound. + +#### Multi-Column-Family Transactions + +A transaction can write to several column families and commits them atomically: + +```rust +let users = db.get_column_family("users")?; +let orders = db.get_column_family("orders")?; + +let mut txn = db.begin_transaction()?; +txn.put(&users, b"u1", b"alice")?; +txn.put(&orders, b"o1", b"u1")?; +txn.commit()?; +``` + +#### Rollback + +```rust +let mut txn = db.begin_transaction()?; +txn.put(&cf, b"k", b"v")?; +txn.rollback()?; // nothing is written +``` + +Dropping a transaction without committing also rolls it back. + +### Transactions + +#### Isolation Levels + +```rust +use tidesdb::IsolationLevel; + +let txn = db.begin_transaction_with_isolation(IsolationLevel::Serializable)?; +``` + +| Level | Behavior | +|-------|----------| +| `IsolationLevel::ReadUncommitted` | Every version is visible, including sequences still in progress | +| `IsolationLevel::ReadCommitted` | The newest committed version, re-read on every operation (default) | +| `IsolationLevel::RepeatableRead` | Frozen snapshot; commit validates that every key read is unchanged and that no key appeared inside a scanned range | +| `IsolationLevel::Snapshot` | Frozen snapshot; commit claims each written key first-committer-wins | +| `IsolationLevel::Serializable` | Both of the checks above | + +`begin_transaction()` uses `ReadCommitted`. `begin_transaction_cf(&cf)` uses the column family's configured `default_isolation_level`: + +```rust +use tidesdb::{ColumnFamilyConfig, IsolationLevel}; + +db.create_column_family( + "rr", + ColumnFamilyConfig::default().default_isolation_level(IsolationLevel::RepeatableRead), +)?; +let cf = db.get_column_family("rr")?; +let txn = db.begin_transaction_cf(&cf)?; // RepeatableRead +``` + +#### Handling Conflicts + +`commit` returns `ErrorCode::Conflict` when another transaction committed a conflicting write first. Even at `ReadCommitted`, a commit is refused with `Conflict` if it writes a key that a prepared but not yet resolved two-phase transaction read. Retry the whole transaction: + +```rust +use tidesdb::IsolationLevel; + +loop { + let mut txn = db.begin_transaction_with_isolation(IsolationLevel::Snapshot)?; + let n: u64 = match txn.get(&cf, b"counter") { + Ok(v) => u64::from_be_bytes(v.try_into().unwrap()), + Err(e) if e.is_not_found() => 0, + Err(e) => return Err(e), + }; + txn.put(&cf, b"counter", &(n + 1).to_be_bytes())?; + match txn.commit() { + Ok(()) => break, + Err(e) if e.is_conflict() => continue, + Err(e) => return Err(e), + } +} +``` + +`DbStats::txn_conflicts` counts commits and prepares refused with a conflict. + +#### Savepoints + +Savepoints allow partial rollback within a transaction: + +```rust +let mut txn = db.begin_transaction()?; +txn.put(&cf, b"key1", b"value1")?; + +txn.savepoint("sp1")?; +txn.put(&cf, b"key2", b"value2")?; + +// key2 is discarded, key1 remains +txn.rollback_to_savepoint("sp1")?; + +txn.savepoint("sp2")?; +txn.release_savepoint("sp2")?; // forget it without rolling back + +txn.put(&cf, b"key3", b"value3")?; +txn.commit()?; // writes key1 and key3 +``` + +Rolling back to or releasing a savepoint that does not exist returns `ErrorCode::NotFound`. Rolling back to a savepoint keeps that savepoint and discards any created after it. + +#### Transaction Reset + +`reset` discards a transaction's state, committed or not, and begins it again at a new isolation level and a fresh snapshot. It also restores the database's default timeout and clears any pending abort request. The `Transaction` value is reused, but the engine allocates a new transaction internally, so this is a convenience, not an optimisation: + +```rust +use tidesdb::IsolationLevel; + +let mut txn = db.begin_transaction()?; +txn.put(&cf, b"a", b"1")?; +txn.commit()?; + +txn.reset(IsolationLevel::Serializable)?; +txn.put(&cf, b"b", b"2")?; +txn.commit()?; +``` + +#### Transaction State + +```rust +use tidesdb::TxnState; + +let mut txn = db.begin_transaction()?; +assert_eq!(txn.state()?, TxnState::Active); +txn.put(&cf, b"k", b"v")?; +txn.commit()?; +assert_eq!(txn.state()?, TxnState::Committed); +``` + +| `TxnState` | Meaning | +|------------|---------| +| `Active` | Buffering writes, not yet resolved | +| `Prepared` | Durably prepared under an xid, awaiting commit or rollback | +| `Committed` | Committed and applied | +| `Aborted` | Rolled back, aborted or expired | + +#### Timeouts + +A transaction can be bounded in time, either for every transaction with `Config::txn_timeout_seconds`, or for one transaction with `set_timeout`, which overrides the database default: + +```rust +use std::time::Duration; +use tidesdb::ErrorCode; + +let mut txn = db.begin_transaction()?; +txn.set_timeout(Some(Duration::from_secs(5)))?; // measured from this call +txn.put(&cf, b"k", b"v")?; + +// ... later ... +match txn.commit() { + Err(e) if e.code() == Some(ErrorCode::TxnExpired) => println!("took too long"), + other => other?, +} + +// `None` clears the timeout +let mut txn = db.begin_transaction()?; +txn.set_timeout(None)?; +``` + +The transaction is not aborted in the background: its next operation notices the deadline and fails with `ErrorCode::TxnExpired` (until then `state()` still reports `Active`). Sub-second durations round up to one second. `set_timeout(None)` removes the timeout altogether; it does not fall back to `Config::txn_timeout_seconds`. + +:::note[Timing] +Deadlines are checked against the engine's one-second clock, so expiry can land up to about two seconds after the nominal deadline. +::: + +#### Aborting from Another Thread + +A `Transaction` may be moved to another thread but not shared between threads. To abort it from elsewhere, take an `AbortHandle`. The handle is `Send + Sync` and `Clone`, and may outlive the transaction, but it borrows the database, so it cannot be passed to `std::thread::spawn` (which needs `'static`). Use `std::thread::scope`: + +```rust +use tidesdb::ErrorCode; + +let mut txn = db.begin_transaction()?; +txn.put(&cf, b"k", b"v")?; + +let handle = txn.abort_handle(); +std::thread::scope(|s| { + s.spawn(move || handle.request_abort()); +}); + +let err = txn.commit().unwrap_err(); +assert_eq!(err.code(), Some(ErrorCode::TxnAborted)); +``` + +`request_abort` only sets a flag: the owning thread's next operation on the transaction (including `commit`) aborts it and returns `ErrorCode::TxnAborted`, which is distinct from the engine's own `Conflict`. A transaction that has already prepared is not affected, and aborting a committed transaction does nothing. + +### Snapshots and Point-in-Time Reads + +#### Snapshots + +A `Snapshot` names a point in time that transactions can read at: + +```rust +let snap = db.snapshot()?; +println!("snapshot at sequence {}", snap.seq()); + +// ... more writes happen ... + +let old = db.begin_transaction_at_snapshot(&snap)?; +let value = old.get(&cf, b"k")?; // value as of the snapshot +``` + +The transaction runs at `RepeatableRead` with its snapshot pinned to the snapshot's sequence. It is not read-only: writes are accepted, and its commit validates its reads against that older point in time, so it is likely to conflict if anything it read has changed since. The transaction borrows the snapshot, which must outlive it. A snapshot is `Send + Sync`, so several threads can begin transactions at the same one: + +```rust +let snap = db.snapshot()?; +std::thread::scope(|s| { + for _ in 0..4 { + s.spawn(|| { + let txn = db.begin_transaction_at_snapshot(&snap).unwrap(); + let _ = txn.get(&cf, b"k"); + }); + } +}); +``` + +:::caution[Cost] +While a snapshot lives, the versions it can see are protected from compaction, which costs the same as a long-running transaction. Drop it as soon as the point in time is no longer needed. +::: + +#### Reading at a Sequence Number + +Every transaction reads below a sequence ceiling, returned by `read_snapshot`: + +- `u64::MAX` for `ReadUncommitted` +- the current sequence for `ReadCommitted` +- the sequence frozen at begin for `RepeatableRead` and stronger + +That sequence can be used later to begin a transaction that reads at the same point (a `RepeatableRead` transaction, like one begun at a `Snapshot`): + +```rust +use tidesdb::{ErrorCode, IsolationLevel}; + +// hold the point in time open with a repeatable read transaction +let holder = db.begin_transaction_with_isolation(IsolationLevel::RepeatableRead)?; +let seq = holder.read_snapshot(); + +// ... more writes happen ... + +assert!(db.oldest_readable_seq() <= seq); +let at = db.begin_transaction_at_seq(seq)?; +let value = at.get(&cf, b"k")?; +``` + +`begin_transaction_at_seq` returns `ErrorCode::TooOld` once compaction has discarded the versions that sequence resolves to. A sequence stays readable only while an open transaction or a `Snapshot` holds it. `oldest_readable_seq()` is the oldest sequence it still accepts; it only rises while the database is open. It is not persisted: after a reopen it starts again from 0, so do not keep sequence numbers across a restart and expect `TooOld` to protect you. A sequence above the current one is treated as the current one. + +### Two-Phase Commit + +`prepare` runs the commit's conflict checks and durably logs the write batch under a transaction id (`xid`), leaving the writes invisible until `commit_prepared` or `rollback_prepared`: + +```rust +use tidesdb::TxnState; + +let mut txn = db.begin_transaction()?; +txn.put(&cf, b"k", b"v")?; + +txn.prepare(b"xid-1")?; +assert_eq!(txn.state()?, TxnState::Prepared); + +// ... coordinator decides ... +txn.commit_prepared()?; // or: txn.rollback_prepared()?; +``` + +`commit_prepared` on a transaction that was not prepared returns `ErrorCode::InvalidArgs`. A transient I/O failure in `commit_prepared` leaves the transaction prepared so the call can be retried. The xid must be non-empty. + +#### Recovering Prepared Transactions + +Dropping a prepared transaction without resolving it leaves it in doubt on disk. After the next open, `recover_prepared` lists every such transaction as a `PreparedTransaction { xid, txn }`, with `txn` in the `Prepared` state: + +```rust +let db = TidesDB::open(Config::new("./mydb"))?; + +// `should_commit` stands for your coordinator's decision for this xid +for mut p in db.recover_prepared()? { + if should_commit(&p.xid) { + p.txn.commit_prepared()?; + } else { + p.txn.rollback_prepared()?; + } +} +``` + +:::caution[Resolve each entry once] +The set is fixed when the database opens: calling `recover_prepared` again before reopening lists the same transactions, including ones already resolved. Resolve each entry once; committing a stale entry applies its batch again, over any newer writes. +::: + +### Iterating Over Data + +Iterators are bidirectional cursors over a column family at the transaction's snapshot. Positioning calls (`seek_to_first`, `seek_to_last`, `seek`, `seek_for_prev`, `next`, `prev`) succeed when they run off either end; check `is_valid()` to see whether the iterator is on an entry. `seek` and `seek_for_prev` with an empty key fail with `InvalidArgs`. + +#### Forward Iteration + +```rust +let txn = db.begin_transaction()?; +let mut iter = txn.new_iterator(&cf)?; + +iter.seek_to_first()?; +while iter.is_valid() { + let (key, value) = iter.key_value()?; + println!("{:?} = {:?}", key, value); + iter.next()?; +} +``` + +#### Backward Iteration + +```rust +let mut iter = txn.new_iterator(&cf)?; +iter.seek_to_last()?; +while iter.is_valid() { + let key = iter.key()?; + println!("{:?}", key); + iter.prev()?; +} +``` + +#### Seek Operations + +```rust +let mut iter = txn.new_iterator(&cf)?; + +// first key >= b"user:" +iter.seek(b"user:")?; +while iter.is_valid() { + let key = iter.key()?; + if !key.starts_with(b"user:") { + break; + } + println!("{}", String::from_utf8_lossy(&key)); + iter.next()?; +} + +// last key <= b"m" +iter.seek_for_prev(b"m")?; +if iter.is_valid() { + println!("{:?}", iter.key()?); +} +``` + +#### Iterator Methods + +| Method | Description | +|--------|-------------| +| `seek_to_first()` | Position at the first key | +| `seek_to_last()` | Position at the last key | +| `seek(key)` | Position at the first key >= `key` | +| `seek_for_prev(key)` | Position at the last key <= `key` | +| `next()` / `prev()` | Move to the next / previous entry | +| `is_valid()` | Whether the iterator is on an entry | +| `key()` | `Result>`, the current key | +| `value()` | `Result>`, the current value | +| `key_value()` | `Result<(Vec, Vec)>`, both in one call | +| `expires_at()` | `Result>`, the current entry's absolute expiry in Unix seconds, `None` if it never expires | + +Calling `key()`, `value()` or the others when the iterator is not valid returns an error (`NotFound`). + +#### Range Iterators + +`new_range_iterator(&cf, lower, upper)` only opens the sstables overlapping `[lower, upper]` (both bounds inclusive; the memtables are always included), which is much cheaper than `new_iterator` for a narrow scan of a large family: + +```rust +let txn = db.begin_transaction()?; +let mut iter = txn.new_range_iterator(&cf, b"k010", b"k019")?; + +iter.seek(b"k010")?; +while iter.is_valid() { + let key = iter.key()?; + if key.as_slice() > b"k019".as_slice() { + break; // stop at the upper bound yourself + } + println!("{}", String::from_utf8_lossy(&key)); + iter.next()?; +} +``` + +:::caution[A promise, not a fence] +The range is a promise, not a fence: results are only defined inside it, and seeking or stepping outside it may report a key that exists as absent. Seek inside the range and stop at the bound yourself. +::: + +#### Iterators and Commit + +An iterator borrows its transaction, and `commit` takes `&mut self`, so the iterator must be dropped before the transaction commits. Scope it: + +```rust +let mut txn = db.begin_transaction()?; +{ + let mut iter = txn.new_iterator(&cf)?; + iter.seek_to_first()?; + while iter.is_valid() { + // ... inspect entries ... + iter.next()?; + } +} // iter dropped +txn.put(&cf, b"summary", b"done")?; +txn.commit()?; +``` + +At `RepeatableRead` and `Serializable`, a scan is also what records the scanned range for conflict detection. The range is recorded when the iterator is dropped, which is one more reason to drop it before committing. + +### Commit Hooks (Change Data Capture) + +A commit hook is a callback that receives every committed batch for a column family: + +```rust +use std::sync::{Arc, Mutex}; +use tidesdb::CommitOp; + +let log: Arc>> = Arc::default(); +let sink = Arc::clone(&log); + +cf.set_commit_hook(move |ops: &[CommitOp], commit_seq: u64| { + for op in ops { + if op.is_delete { + println!("[{commit_seq}] delete {:?}", op.key); + } else { + println!("[{commit_seq}] put {:?} = {:?} (expires {:?})", op.key, op.value, op.expires_at); + } + } + sink.lock().unwrap().extend_from_slice(ops); + 0 // non-zero reports a failure +})?; + +// ... commits ... + +cf.clear_commit_hook()?; +``` + +`CommitOp` fields: + +| Field | Type | Description | +|-------|------|-------------| +| `key` | `Vec` | The key, or the inclusive lower bound of a range delete | +| `value` | `Option>` | The value; `None` for every kind of delete | +| `range_end` | `Option>` | The exclusive upper bound of a range or prefix delete (every key in `[key, range_end)` was deleted); `None` otherwise | +| `expires_at` | `Option` | Absolute expiry in Unix seconds as the engine stored it (not the `Duration` passed to `put_with_ttl`); `None` if it never expires | +| `is_delete` | `bool` | Whether this is a delete (point, range or prefix) | + +How each write arrives: + +| Write | `key` | `value` | `range_end` | `is_delete` | +|-------|-------|---------|-------------|-------------| +| `put`, `put_with_ttl` | the key | `Some(value)` | `None` | `false` | +| `delete`, `single_delete` | the key | `None` | `None` | `true` | +| `delete_range(lo, Some(hi))` | `lo` | `None` | `Some(hi)` | `true` | +| `delete_prefix(p)` | `p` | `None` | `Some(first key past p)`, e.g. `b"user;"` for `b"user:"` | `true` | +| `delete_range(lo, None)` | `lo` | `None` | `None` | `true` | + +A consumer replicating the hook's ops applies a range delete as "delete every key in `[key, range_end)`". + +Behavior: + +- The hook fires synchronously on the committing thread after every commit (and every `commit_prepared`) that touches the family, with the family's whole batch, once the commit is logged, applied and visible. It is not a durability signal: under `SyncMode::None` or `SyncMode::Interval` the write-ahead log may not be on disk yet. +- Each commit gets a larger `commit_seq` than the one before it, but commits on different threads may run the hook concurrently, so calls can arrive out of `commit_seq` order. The closure must be `Fn + Send + Sync + 'static`. +- A range delete with no upper bound (and a prefix delete of a prefix made only of `0xFF` bytes, which has no key past it) arrives exactly like a point delete of its lower bound: the C library passes nothing that tells them apart. A `single_delete` arrives like a `delete`. +- A non-zero return is logged by the engine but does not undo the commit. A panic is caught and counts as a failure. +- The hook belongs to the column family, not the handle: it stays installed after the handle is dropped, until it is replaced, cleared, or the database closes. `set_commit_hook` replaces any existing hook; replaced hooks are kept alive until close. + +## Maintenance + +### Memtable Flush + +The memtable is shared by every column family, so flushing is a database operation: + +```rust +db.flush_memtable()?; // rotate and flush; waits up to 60 s for the queue to drain +if db.is_flushing() { + println!("a flush is in progress"); +} +``` + +`flush_memtable` returns `ErrorCode::Locked` if the flush queue has not drained after 60 seconds; the flush itself carries on. `is_flushing` is true while an immutable memtable is queued or being flushed. + +### Compaction + +```rust +use tidesdb::ErrorCode; + +// one forced compaction pass, merging even when no trigger is due +match cf.compact() { + Ok(()) => {} + Err(e) if e.is_locked() => println!("a compaction is already running"), + Err(e) => return Err(e), +} + +// compact every sstable overlapping [b"a", b"n") +cf.compact_range(Some(b"a"), Some(b"n"))?; + +// from b"m" to the end +cf.compact_range(Some(b"m"), None)?; + +if cf.is_compacting() { + println!("compaction in progress"); +} +``` + +Both calls run synchronously, and both return `ErrorCode::Locked` without waiting if the family is already compacting or being reconfigured or renamed; retry later. `compact_range` also returns `Locked` if the sstable layout changes while it plans, and does nothing (successfully) when fewer than two sstables overlap the range. `compact_range(None, None)` is rejected with `InvalidArgs`; use `compact()` instead. `is_compacting` reports the same claim, so it is also true during a runtime configuration update or a rename. + +### WAL Sync + +```rust +db.sync_wal()?; +``` + +Forces an fsync of the write-ahead log, for explicit durability control under `SyncMode::None` or `SyncMode::Interval`. + +### Checkpoint + +```rust +db.checkpoint()?; +``` + +`checkpoint` is a durability barrier: it flushes the memtable to L1, then forces the value log, write-ahead log and manifest to disk regardless of the configured sync mode. It does not write a copy anywhere; use `backup` for that. + +### Backup + +```rust +db.backup("./mydb-backup")?; + +// the backup is a regular database directory +let restored = TidesDB::open(Config::new("./mydb-backup"))?; +``` + +Writes a consistent, directly openable copy of the database into the directory, which must be absent or empty. A non-empty directory is refused with `ErrorCode::Io`; the parent directory must already exist. + +### Runtime Configuration Updates + +```rust +use tidesdb::ColumnFamilyConfig; + +let updated = ColumnFamilyConfig::default() + .l1_file_count_trigger(8) + .bloom_fpr(0.005); +cf.update_runtime_config(&updated, true)?; // true = persist to the manifest +``` + +Every `ColumnFamilyConfig` field can be changed at runtime, since keys are always ordered byte-wise and sstables stay mergeable. With `persist_to_disk = true` the configuration is written to the manifest; with `false` it lasts until the database is reopened, unless a later `rename_column_family` writes the family's manifest entry, which then saves the in-memory configuration too. Start from `cf.get_stats()?.config` to change one field and keep the rest. + +The update returns `ErrorCode::Locked` without waiting if the family is compacting or under another reconfiguration or rename. The new configuration is validated as on create; `InvalidArgs` means one of: `level_size_ratio` below 2, `min_levels` of 0 or less, `bloom_fpr` outside `(0, 1)` with bloom filters enabled, `tombstone_density_trigger` outside `[0, 1]`, or a codec this build does not support. + +### Open File Limit + +```rust +let ceiling = tidesdb::raise_open_file_limit(8192); +println!("open-file ceiling is now {ceiling}"); +let db = TidesDB::open(Config::new("./mydb").max_open_sstables(4096))?; +``` + +TidesDB reads the process's open-file ceiling at open and lowers `max_open_sstables` to fit it. `raise_open_file_limit(desired)` raises the ceiling toward `desired` and returns the ceiling in effect afterwards; call it before `TidesDB::open`. Passing `0` or a negative value only reports the current ceiling. A failed or partial raise is not an error. + +## Statistics + +### Column Family Statistics + +```rust +let stats = cf.get_stats()?; + +println!("levels: {}", stats.num_levels); +println!("keys: {} (unflushed: {})", stats.total_keys, stats.unflushed_key_count); +println!("data size: {} bytes", stats.total_data_size); +println!("read amp: {:.2}", stats.read_amp); +println!("tombstone ratio: {:.2}", stats.tombstone_ratio); + +for (i, size) in stats.level_sizes.iter().enumerate() { + println!( + "L{}: {} bytes, {} sstables, {} keys", + i + 1, + size, + stats.level_num_sstables[i], + stats.level_key_counts[i] + ); +} + +println!("config: {:?}", stats.config); +``` + +`CfStats` fields: + +| Field | Type | Description | +|-------|------|-------------| +| `name` | `String` | The family's current name (reflects renames, unlike `ColumnFamily::name()`) | +| `num_levels` | `i32` | Deepest level that currently holds an sstable (0 for a family with nothing flushed) | +| `config` | `ColumnFamilyConfig` | A copy of the column family configuration | +| `level_sizes` | `Vec` | On-disk size of each level; index 0 is L1 | +| `level_num_sstables` | `Vec` | Sstable count of each level | +| `level_key_counts` | `Vec` | Key count of each level | +| `level_tombstone_counts` | `Vec` | Tombstone count of each level | +| `total_keys` | `u64` | Sum of each sstable's distinct key count (a key present in several sstables counts once per sstable) | +| `total_data_size` | `u64` | The family's own on-disk size (sum of its key logs); separated values are counted in `DbStats::vlog_file_size` | +| `avg_key_size` | `f64` | Average key length in bytes | +| `avg_value_size` | `f64` | Average value length in bytes | +| `read_amp` | `f64` | Point-lookup read amplification (sstables a worst-case get may probe) | +| `btree_total_nodes` | `u64` | Total btree nodes | +| `btree_max_height` | `u32` | Maximum btree height | +| `btree_avg_height` | `f64` | Average btree height | +| `total_tombstones` | `u64` | Total tombstones across every sstable | +| `tombstone_ratio` | `f64` | `total_tombstones / total_keys` | +| `max_sst_density` | `f64` | Worst per-sstable tombstone density observed | +| `max_sst_density_level` | `i32` | 1-based level of `max_sst_density`, 0 if none | +| `wal_bytes_written` | `u64` | This family's attributed share of write-ahead log bytes | +| `flush_bytes_written` | `u64` | On-disk bytes this family's flushes wrote | +| `compaction_bytes_written` | `u64` | On-disk bytes this family's compactions wrote | +| `compaction_bytes_read` | `u64` | On-disk bytes this family's compactions read | +| `user_bytes_written` | `u64` | Logical key+value bytes committed to this family | +| `compaction_count` | `u64` | Compactions this family has run | +| `unflushed_key_count` | `u64` | Distinct keys still only in the shared memtables | +| `filter_resident_bytes` | `u64` | Memory held by this family's bloom filter directories outside the block cache | + +The `level_*` vectors all have `num_levels` entries (empty when nothing is flushed); a level between L1 and the deepest one that holds nothing shows as zeros. Memtable figures are database-level and live in `DbStats`. + +### Cardinality and Range Statistics + +```rust +let distinct = cf.estimate_cardinality()?; +println!("about {distinct} distinct keys"); + +let rs = cf.range_stats(b"k000", b"k100")?; // [key_a, key_b) +println!( + "{} sorted runs, {} keys ({})", + rs.sstables_overlapping, + rs.estimated_keys, + if rs.keys_exact { "exact" } else { "estimated" } +); +``` + +`range_stats` describes the key range `[key_a, key_b)` for a query planner. `RangeStats` fields: + +| Field | Type | Description | +|-------|------|-------------| +| `sstables_overlapping` | `u64` | Sstables overlapping the range, counting each one even when several share a level | +| `estimated_keys` | `u64` | Live keys the range holds | +| `keys_exact` | `bool` | Whether `estimated_keys` was counted exactly rather than estimated | + +### Database Statistics + +```rust +let s = db.get_db_stats()?; +println!("column families: {}", s.num_column_families); +println!("sequence: {}", s.global_seq); +println!("commits: {} conflicts: {}", s.txn_commits, s.txn_conflicts); +println!("memtable: {} bytes, flushing: {}", s.memtable_bytes, s.is_flushing); +println!("sstables: {}, open: {}", s.total_sstable_count, s.num_open_sstables); +println!("vlog: {} bytes in {} segments", s.vlog_file_size, s.vlog_segment_count); +``` + +`DbStats` fields: + +| Field | Type | Description | +|-------|------|-------------| +| `num_column_families` | `i32` | Number of column families | +| `immutable_memtable_count` | `i32` | Immutable memtables awaiting flush (the L0 queue depth) | +| `compaction_pending_count` | `i32` | Compaction jobs queued | +| `total_sstable_count` | `i32` | Total sstables across every column family and level | +| `total_data_size_bytes` | `u64` | On-disk key-log bytes across every column family | +| `num_open_sstables` | `i32` | Currently open sstable file handles | +| `global_seq` | `u64` | Current global sequence number (the MVCC clock) | +| `min_snapshot_seq` | `u64` | Oldest frozen snapshot any live transaction holds | +| `active_txn_count` | `i32` | Live registered transactions (read committed and stronger) | +| `txn_memory_bytes` | `i64` | Bytes held by in-flight transactions | +| `txn_commits` | `u64` | Transactions committed since open | +| `txn_conflicts` | `u64` | Commits and prepares refused with a conflict since open | +| `memtable_bytes` | `usize` | Memory the active memtable occupies | +| `is_flushing` | `bool` | Whether an immutable memtable is queued or flushing | +| `next_cf_index` | `u32` | Next column family id to be assigned | +| `wal_generation` | `u64` | Generation of the active write-ahead log | +| `flush_count` | `u64` | Sstables flushed across every column family | +| `compaction_count` | `u64` | Compactions run across every column family | +| `flush_bytes_written` | `u64` | Flush output bytes | +| `compaction_bytes_written` | `u64` | Compaction output bytes | +| `compaction_bytes_read` | `u64` | Compaction input bytes | +| `wal_bytes_written` | `u64` | Framed write-ahead log bytes, measured at the append | +| `user_bytes_written` | `u64` | Logical committed bytes | +| `vlog_file_size` | `u64` | Total value-log file size | +| `vlog_value_count` | `u64` | Values currently indexed in the value log | +| `vlog_used_bytes` | `u64` | Uncompressed length the indexed values represent | +| `vlog_stored_bytes` | `u64` | On-disk length of the indexed values | +| `vlog_live_bytes` | `u64` | Value-log bytes the live sstables still reference | +| `vlog_segment_count` | `u64` | Open value-log segment files | +| `vlog_bytes_written` | `u64` | Value-log bytes ever appended, reclamation rewrites included | +| `vlog_dead_bytes` | `u64` | Value-log bytes beyond what live values account for | +| `vlog_reclaim_calls` | `u64` | Value-log reclaims attempted | +| `vlog_reclaim_passes` | `u64` | Value-log reclaim passes that drained a segment | +| `vlog_segments_retired` | `u64` | Value-log segment files a reclaim has unlinked | +| `vlog_segments_drainable` | `u64` | Sealed segments worth draining | +| `writes_throttled` | `u64` | Commits the L0 admission policy made dwell | +| `writes_blocked` | `u64` | Commits the L0 admission policy made wait for the flush queue | +| `write_stall_us` | `u64` | Total microseconds commits were held in L0 admission | +| `write_stall_ceiling_hits` | `u64` | Commits admitted only because the admission wait ceiling expired | + +### Cache Statistics + +```rust +let c = db.get_cache_stats()?; +if c.enabled { + println!( + "{} entries, {} bytes, hit rate {:.1}% ({} hits / {} misses), {} partitions", + c.total_entries, c.total_bytes, c.hit_rate * 100.0, c.hits, c.misses, c.num_partitions + ); +} +``` + +| Field | Type | Description | +|-------|------|-------------| +| `enabled` | `bool` | Whether the cache is enabled | +| `total_entries` | `usize` | Number of cached entries | +| `total_bytes` | `usize` | Bytes used by the cache | +| `hits` | `u64` | Cache hits | +| `misses` | `u64` | Cache misses | +| `hit_rate` | `f64` | `hits / (hits + misses)`, 0 before any lookup | +| `num_partitions` | `usize` | Number of cache shards | + +### Stall Statistics + +`get_stall_stats` reports where writers have been made to wait since the database opened. Each reason is a `StallStat { count, total_us, max_us }` (number of waits, summed wait in microseconds, longest single wait in microseconds): + +```rust +let stalls = db.get_stall_stats()?; +for (reason, stat) in stalls.iter() { + println!("{reason:>16}: {} waits, {} us total, {} us max", stat.count, stat.total_us, stat.max_us); +} +println!("admission waits: {}", stalls.admission.count); +``` + +| `StallStats` field | Description | +|--------------------|-------------| +| `wal_append` | Waiting on the write-ahead log (staging space or, under a syncing mode, the write) | +| `rotate_lock` | Waiting to take the rotation lock while another committer rotated | +| `rotate_work` | Performing a memtable rotation on everyone's behalf | +| `admission` | Held by write admission because the unflushed backlog was too deep | +| `manifest_commit` | Inside a manifest commit | + +`iter()` yields every reason with the library's stable short name, for logging or a stats table. + +### I/O Statistics + +`get_io_stats` reports what each class of file asked of the device since the database opened. Each class is an `IoStat { ops, bytes, total_us, max_us }` (writes issued, bytes written, summed time inside those writes, slowest single write; times in microseconds): + +```rust +let io = db.get_io_stats()?; +for (class, stat) in io.iter() { + println!("{class:>8}: {} writes, {} bytes, {} us max", stat.ops, stat.bytes, stat.max_us); +} +``` + +| `IoStats` field | Description | +|-----------------|-------------| +| `sstable` | Key logs, written by flush and compaction | +| `wal` | The write-ahead log | +| `vlog` | Value log segments | + +`iter()` yields every class with the library's stable short name. + +### Encoding Statistics + +`get_klog_encoding_stats` reports what each encoding chain achieved on the key logs, one entry per chain the live sstables were written with. `get_vlog_encoding_stats` does the same for the separated values in the value log. + +```rust +use tidesdb::CompressionAlgorithm; + +for e in db.get_klog_encoding_stats()? { + let chain: Vec<_> = e.ids.iter().map(|id| CompressionAlgorithm::from_id(*id)).collect(); + println!( + "{:?}: {} -> {} bytes over {} items, ratio {:.2}", + chain, e.logical_bytes, e.stored_bytes, e.item_count, e.ratio() + ); +} +``` + +| `EncodingStats` field | Type | Description | +|-----------------------|------|-------------| +| `ids` | `Vec` | Encoding ids in the order applied; empty when stored verbatim. Ids `1..=4` are `CompressionAlgorithm` values | +| `logical_bytes` | `u64` | Size of the data before encoding | +| `stored_bytes` | `u64` | Size of the data on disk | +| `item_count` | `u64` | Values for the value log, sstables for the key logs | + +`ratio()` returns `logical_bytes / stored_bytes`, or 0 when nothing is stored. + +## Configuration + +### Sync Modes + +The write-ahead log is shared, so durability is set once per database with `Config::memtable_sync_mode`: + +```rust +use tidesdb::{Config, SyncMode}; + +let config = Config::new("./mydb") + .memtable_sync_mode(SyncMode::Interval) + .memtable_sync_interval_us(100_000); // 100 ms +``` + +| `SyncMode` | Behavior | +|------------|----------| +| `SyncMode::None` | Nothing at commit; fastest, a crash of the process or the machine may lose commits (default) | +| `SyncMode::Full` | fsync every commit; slowest, no commit loss on crash | +| `SyncMode::Interval` | fsync on a background interval (`memtable_sync_interval_us`); only a machine crash can lose the last interval | + +Under `None` or `Interval`, call `db.sync_wal()` or `db.checkpoint()` to force durability at a point of your choosing. + +### Compression + +Compression is configured per column family as an **encoding pipeline**: up to eight stages applied in order to key-log nodes and separated values. The default pipeline is empty, which stores data verbatim (no compression). + +```rust +use tidesdb::{ColumnFamilyConfig, CompressionAlgorithm}; + +// one stage +let lz4 = ColumnFamilyConfig::new().compression_algorithm(CompressionAlgorithm::Lz4); + +// explicit pipeline +let zstd = ColumnFamilyConfig::new().encoding_pipeline(vec![CompressionAlgorithm::Zstd]); + +// no compression (clears the pipeline) +let plain = ColumnFamilyConfig::new().compression_algorithm(CompressionAlgorithm::None); +``` + +| `CompressionAlgorithm` | Cargo feature | +|------------------------|---------------| +| `None` | always available | +| `Snappy` | `snappy` | +| `Lz4` | `lz4` | +| `Lz4Fast` | `lz4` | +| `Zstd` | `zstd` | + +Every variant is always defined, but a codec is only usable when the C library was built with it. Check before choosing one: + +```rust +use tidesdb::{ColumnFamilyConfig, CompressionAlgorithm}; + +let algo = if CompressionAlgorithm::Zstd.is_available() { + CompressionAlgorithm::Zstd +} else { + CompressionAlgorithm::None +}; +db.create_column_family("logs", ColumnFamilyConfig::new().compression_algorithm(algo))?; +``` + +`CompressionAlgorithm::from_id(u8)` maps an encoding id (as found in `EncodingStats::ids`) back to the algorithm. A pipeline of more than eight stages is rejected with `InvalidArgs` when the column family is created or updated; `None` entries in a pipeline are skipped. + +### Log Levels + +| `LogLevel` | Emits | +|------------|-------| +| `LogLevel::None` | Nothing | +| `LogLevel::Trace` | Highly detailed messages for technical debugging, and everything below | +| `LogLevel::Info` | Engine status and operations, and everything below (default) | +| `LogLevel::Warn` | Non-imminent errors that require awareness, and errors | +| `LogLevel::Error` | Failed operations only | + +Logs go to stderr unless `log_to_file(true)` is set, in which case they go to `/LOG`. Logging is process-wide: the level is set by the most recently opened database, and the last database opened with `log_to_file` owns the log file until it closes. `log_truncation_at` only applies with `log_to_file`. + +## Database Configuration Reference + +`Config` holds the settings shared by every column family: the memtable, write-ahead log, value log, block cache and worker pool. Every field is public and has a builder method of the same name. `Config::new(path)` and `Config::default()` take their values from the C library's defaults; `Config::default()` leaves `db_path` empty. + +A numeric field set to `0` is resolved to the library default at open for the thread counts, `block_cache_size`, `max_open_sstables`, `value_separation_threshold`, `vlog_segment_size`, `memtable_write_buffer_size` and the memtable skip-list settings. + +| Field / builder | Type | Default | Description | +|-----------------|------|---------|-------------| +| `db_path` | `String` | set by `Config::new` | Path to the database directory | +| `num_flush_threads(n)` | `i32` | 2 | Number of flush worker threads | +| `num_compaction_threads(n)` | `i32` | 2 | Number of compaction worker threads | +| `log_level(level)` | `LogLevel` | `Info` | Minimum severity to log (process-wide; the last database opened sets it) | +| `block_cache_size(size)` | `usize` | 64 MiB | Size in bytes of the block cache for hot sstable blocks | +| `max_open_sstables(n)` | `usize` | 1024 | Maximum concurrently open sstable file handles; lowered at open to fit the process's open-file ceiling (see `raise_open_file_limit`) | +| `log_to_file(enable)` | `bool` | `false` | Write the log to `/LOG` instead of stderr (process-wide sink) | +| `log_truncation_at(size)` | `usize` | 0 | Size in bytes past which the log file is truncated (0 = never; only with `log_to_file`) | +| `memtable_write_buffer_size(size)` | `usize` | 64 MiB | Memory the active memtable may occupy before it is rotated (0 = auto) | +| `memtable_skip_list_max_level(level)` | `i32` | 12 | Memtable skip list max level (0 = default) | +| `memtable_skip_list_probability(prob)` | `f32` | 0.25 | Memtable skip list level probability (0 = default) | +| `memtable_sync_mode(mode)` | `SyncMode` | `None` | Durability mode for the write-ahead log | +| `memtable_sync_interval_us(interval)` | `u64` | 1,000,000 (1 s) | fsync interval in microseconds for `SyncMode::Interval` | +| `value_separation_threshold(size)` | `usize` | 1024 | Values at or above this size go to the shared value log (0 = default). Keep it at or under a quarter of a column family's `btree_klog_block_size` | +| `vlog_segment_size(size)` | `usize` | 256 MiB | Size at which the value log seals a segment and opens a new one (0 = default) | +| `memtable_l0_queue_stall_threshold(n)` | `i32` | 16 | Immutable-memtable queue depth at which writes stall for backpressure (0 = never) | +| `memtable_idle_flush_seconds(secs)` | `i32` | 30 | Seconds the active memtable may sit unwritten before it is rotated (0 = never) | +| `txn_timeout_seconds(secs)` | `i64` | 0 | Seconds a transaction may stay active before its next operation expires it (0 = no timeout); override per transaction with `set_timeout` | + +The defaults listed are those of TidesDB 10.2.1 and are read from the library at runtime. + +## Column Family Configuration Reference + +Every field is public and has a builder method of the same name, plus `compression_algorithm(algo)` as a shorthand for a one-stage `encoding_pipeline`. `ColumnFamilyConfig::new()` and `ColumnFamilyConfig::default()` take their values from the C library. Every field can be changed at runtime with `ColumnFamily::update_runtime_config`. + +| Field / builder | Type | Default | Description | +|-----------------|------|---------|-------------| +| `level_size_ratio(ratio)` | `usize` | 10 | Target size ratio between successive levels | +| `min_levels(levels)` | `i32` | 1 | Floor on the level count | +| `dividing_level_offset(offset)` | `i32` | 1 | How far above the largest level the dividing level sits | +| `keep_values_inline(enable)` | `bool` | `false` | Keep every value in the key log regardless of the database's `value_separation_threshold` | +| `btree_klog_block_size(size)` | `usize` | 4096 | Target btree key-log node size in bytes (0 = let the btree choose its own, larger, default) | +| `encoding_pipeline(vec)` | `Vec` | empty | Encodings applied in order to key-log nodes and separated values, at most 8 stages; empty stores data verbatim | +| `compression_algorithm(algo)` | (sets `encoding_pipeline`) | | One-stage pipeline; `CompressionAlgorithm::None` clears it | +| `enable_bloom_filter(enable)` | `bool` | `true` | Build bloom filters for point-get pruning | +| `bloom_fpr(fpr)` | `f64` | 0.01 | Target bloom filter false-positive rate | +| `default_isolation_level(level)` | `IsolationLevel` | `ReadCommitted` | Isolation level for transactions begun with `begin_transaction_cf` | +| `l1_file_count_trigger(n)` | `i32` | 4 | L1 sstable count that triggers compaction (0 = this trigger is off) | +| `tombstone_density_trigger(ratio)` | `f64` | 0.5 | Tombstone density in `[0, 1]` at or above which an sstable escalates compaction (0 = disabled) | +| `tombstone_density_min_entries(n)` | `u64` | 4096 | Minimum entries for an sstable to be judged by the density trigger (0 = no minimum) | + +`ColumnFamilyConfig` implements `PartialEq`, so a configuration read back through `cf.get_stats()?.config` can be compared with the one you created. + +## Error Handling + +All fallible calls return `tidesdb::Result`, an alias for `Result`. + +```rust +use tidesdb::{Error, ErrorCode}; + +match txn.get(&cf, b"key") { + Ok(value) => println!("{value:?}"), + Err(e) if e.is_not_found() => println!("not found"), + Err(Error::TidesDB { code, context }) => { + match code { + ErrorCode::Io => eprintln!("I/O error in {context}"), + ErrorCode::NoSpace => eprintln!("disk full; retry once space is freed"), + _ => eprintln!("{context}: {code}"), + } + } + Err(e) => eprintln!("{e}"), +} +``` + +### `Error` + +| Variant | Description | +|---------|-------------| +| `Error::TidesDB { code: ErrorCode, context: &'static str }` | An error from the TidesDB C library. Displays as `": "` | +| `Error::NullPointer(&'static str)` | The library returned a null handle | +| `Error::InvalidString(Utf8Error)` | Invalid UTF-8 string | +| `Error::NulError(NulError)` | A name or path contained an interior NUL byte | + +| Method | Description | +|--------|-------------| +| `code()` | `Option`, the library error code if the error came from the C library | +| `is_not_found()` | `code() == Some(ErrorCode::NotFound)` | +| `is_conflict()` | `code() == Some(ErrorCode::Conflict)` | +| `is_locked()` | `code() == Some(ErrorCode::Locked)`; transient contention that should be retried | +| `Error::from_code(code, context)` | Build an `Error::TidesDB` from a raw C error code | + +### `ErrorCode` + +`ErrorCode` is `#[non_exhaustive]`: new codes may be added in minor releases, so always include a wildcard arm when matching. Its `Display` uses the library's own message. `ErrorCode::from_code(i32)` converts a raw C code (unrecognised codes map to `Unknown`), and `code()` returns the raw `TDB_ERR_*` value. + +| Variant | C code | Meaning | +|---------|--------|---------| +| `Memory` | -1 | Memory allocation failed | +| `InvalidArgs` | -2 | Invalid arguments provided | +| `NotFound` | -3 | Key or resource not found | +| `Io` | -4 | I/O error occurred | +| `Corruption` | -5 | Data corruption detected | +| `Exists` | -6 | Resource already exists | +| `Conflict` | -7 | Another transaction committed a conflicting write first; retry the transaction | +| `TooLarge` | -8 | A value does not fit the space that has to hold it | +| `MemoryLimit` | -9 | Memory limit exceeded | +| `InvalidDb` | -10 | Invalid database handle, or the database is closing | +| `Unknown` | -11 | Unknown error; also what `ErrorCode::from_code` returns for an unrecognised code | +| `Locked` | -12 | Transient contention: nothing was written, retry. Never means absence or data loss. Also returned when opening a directory another handle has open | +| `ReadOnly` | -13 | Read-only database; defined by TidesDB 10 but not returned by any operation | +| `TxnExpired` | -14 | The transaction outlived its timeout and was aborted | +| `NoSpace` | -15 | The device or filesystem is out of space; retrying succeeds once space is freed | +| `TxnAborted` | -16 | The transaction was aborted from another thread with `AbortHandle::request_abort` | +| `TooOld` | -17 | The requested point in time is no longer reconstructable | + +## Complete Example + +```rust +use std::time::Duration; +use tidesdb::{ + ColumnFamilyConfig, CompressionAlgorithm, Config, IsolationLevel, LogLevel, SyncMode, TidesDB, +}; + +fn main() -> tidesdb::Result<()> { + let config = Config::new("./example_db") + .log_level(LogLevel::Warn) + .memtable_sync_mode(SyncMode::Interval) + .memtable_sync_interval_us(100_000); + let db = TidesDB::open(config)?; + + let algo = if CompressionAlgorithm::Lz4.is_available() { + CompressionAlgorithm::Lz4 + } else { + CompressionAlgorithm::None + }; + db.create_column_family( + "users", + ColumnFamilyConfig::new() + .compression_algorithm(algo) + .bloom_fpr(0.01), + )?; + let cf = db.get_column_family("users")?; + + // write + let mut txn = db.begin_transaction()?; + txn.put(&cf, b"user:1", b"Alice")?; + txn.put(&cf, b"user:2", b"Bob")?; + txn.put_with_ttl(&cf, b"session:abc", b"temp", Duration::from_secs(30))?; + txn.commit()?; + + // read + let txn = db.begin_transaction_with_isolation(IsolationLevel::RepeatableRead)?; + let value = txn.get(&cf, b"user:1")?; + println!("user:1 = {}", String::from_utf8_lossy(&value)); + + // scan + let mut iter = txn.new_iterator(&cf)?; + iter.seek(b"user:")?; + while iter.is_valid() { + let (key, value) = iter.key_value()?; + if !key.starts_with(b"user:") { + break; + } + println!(" {} = {}", String::from_utf8_lossy(&key), String::from_utf8_lossy(&value)); + iter.next()?; + } + drop(iter); + drop(txn); + + // maintenance and stats + db.flush_memtable()?; + let stats = cf.get_stats()?; + println!("levels: {}, keys: {}", stats.num_levels, stats.total_keys); + let db_stats = db.get_db_stats()?; + println!("commits: {}", db_stats.txn_commits); + + Ok(()) +} +``` + +## Thread Safety + +| Type | `Send` | `Sync` | Notes | +|------|--------|--------|-------| +| `TidesDB` | yes | yes | Share by reference (e.g. `std::thread::scope`) or in an `Arc` | +| `ColumnFamily<'db>` | yes | yes | Cheap to `Clone` | +| `Snapshot<'db>` | yes | yes | Many threads may begin transactions at one snapshot | +| `Transaction<'db>` | yes | no | Move it to another thread, but do not share it; abort it from elsewhere with an `AbortHandle` | +| `AbortHandle<'db>` | yes | yes | `Clone`; may outlive its transaction but not the database | +| `Iterator<'txn>` | no | no | Use on the thread that owns the transaction | + +Because handles borrow the database, `std::thread::scope` is the simplest way to share it: + +```rust +use tidesdb::{TidesDB, Config, ColumnFamilyConfig}; + +fn main() -> tidesdb::Result<()> { + let db = TidesDB::open(Config::new("./mydb"))?; + db.create_column_family("my_cf", ColumnFamilyConfig::default())?; + let cf = db.get_column_family("my_cf")?; + + std::thread::scope(|s| { + for t in 0..4 { + let (db, cf) = (&db, &cf); + s.spawn(move || { + for i in 0..100 { + let mut txn = db.begin_transaction().unwrap(); + txn.put(cf, format!("t{t}-{i}").as_bytes(), b"v").unwrap(); + txn.commit().unwrap(); + } + }); + } + }); + + Ok(()) +} +``` + +With `Arc` and `std::thread::spawn`, look up the column family inside each thread (`db.get_column_family(name)`), since a `ColumnFamily` borrowed from one thread's `Arc` cannot be sent to a `'static` thread. + +## Utility Functions + +### `tidesdb::TIDESDB_VERSION` + +The TidesDB C library release this crate binds and bundles (`"10.2.1"`). See [Versioning](#versioning). + +### `tidesdb::TIDESDB_LINKED_VERSION` + +The TidesDB release actually linked: equal to `TIDESDB_VERSION` when the bundled source was built, or the version of the system TidesDB found through pkg-config. + +### `tidesdb::raise_open_file_limit` + +`fn raise_open_file_limit(desired: i64) -> i64`. See [Open File Limit](#open-file-limit). + +## Testing + +```bash +git submodule update --init +cargo test + +# with every compression codec +cargo test --features compression + +# on an alternative allocator (at most one) +cargo test --features jemalloc + +# with output +cargo test -- --nocapture + +# a single test +cargo test test_open_close + +# release optimizations +cargo test --release +``` diff --git a/scripts/sync-versions.py b/scripts/sync-versions.py deleted file mode 100755 index 7171b8c..0000000 --- a/scripts/sync-versions.py +++ /dev/null @@ -1,1215 +0,0 @@ -#!/usr/bin/env -S uv run -# /// script -# requires-python = ">=3.11" -# dependencies = [ -# "tomlkit", -# "requests", -# "InquirerPy", -# ] -# /// - -"""Sync tidesdb version features and source crates. - -Fetches TidesDB releases and crates.io state, presents a plan, updates -Cargo.toml/build.rs so each version feature enables a matching tidesdb source -archive crate, and optionally prepares missing or changed source crates. Prepared -crates are checked with cargo publish --dry-run unless --publish is set. - -Flow: - - start - | - v - fetch TidesDB GitHub releases - | - v - read current Cargo.toml features/default - | - +-- --prepare-local-patches? - | | - | v - | write local source crates for selected versions - | | - | write optional .cargo/config.toml [patch.crates-io] entries - | | - | v - | done - | - v - choose versions/default/source-crate set - | | - | +-- --versions: use CLI inputs and defaults, - | | then survey selected source crates - | | - | +-- no --versions: survey all source crates, - | then use interactive pickers - | - v - print plan - | - +-- any selected crate state unknown? -> abort before writes/publish - | - +-- --dry-run? -> exit after plan - | - v - confirm plan unless --yes - | - +-- not --source-crates-only -> update Cargo.toml, build.rs, and CI default - | - v - root files are current or intentionally skipped - | - +-- no source crates selected? -> done - | - v - prepare each selected source crate in work dir - | - v - cargo publish --dry-run each prepared crate unless --publish is set - | - v - clean temp work dir, or keep explicit/requested work dir - | - v - done -""" - -from __future__ import annotations - -import argparse -import copy -import hashlib -import io -import re -import shutil -import subprocess -import sys -import tarfile -import tempfile -from dataclasses import dataclass, field -from pathlib import Path -from typing import Iterable - -import requests -import tomlkit -from InquirerPy import inquirer - -CARGO_TOML = Path("Cargo.toml") -BUILD_RS = Path("build.rs") -BUILD_AND_TEST_YML = Path(".github/workflows/build_and_test.yml") -LICENSE = Path("LICENSE") -REPO = "tidesdb/tidesdb" -SOURCE_CRATE_INITIAL_VERSION = "0.1.0" -CRATES_IO = "https://crates.io/api/v1/crates" -USER_AGENT = "tidesdb-rs source-crate sync (https://github.com/tidesdb/tidesdb-rs)" - - -@dataclass -class ArchiveInfo: - data: bytes - size: int - sha256: str - - -@dataclass -class SourceCrateState: - status: str - published_version: str | None = None - publish_version: str | None = None - reasons: list[str] = field(default_factory=list) - - @property - def needs_publish(self) -> bool: - return self.status in {"missing", "update"} - - @property - def is_blocking(self) -> bool: - return self.status == "unknown" - - -def versions_from_releases(releases: Iterable[dict]) -> list[str]: - return versions_from_tags(release["tag_name"] for release in releases) - - -def versions_from_tags(tags: Iterable[str]) -> list[str]: - versions = [] - for tag in tags: - if not tag.startswith("v"): - continue - version = tag[1:] - if not is_version(version): - continue - if int(version.split(".")[0]) < 9: - continue - versions.append(version) - - return sorted(set(versions), key=version_key) - - -def fetch_versions_with_gh() -> list[str] | None: - if shutil.which("gh") is None: - return None - - cmd = [ - "gh", - "api", - "--method", - "GET", - "--paginate", - f"repos/{REPO}/releases", - "-f", - "per_page=100", - "--jq", - ".[].tag_name", - ] - try: - output = subprocess.check_output(cmd, text=True, stderr=subprocess.PIPE) - except subprocess.CalledProcessError as exc: - print( - f"warning: gh release query failed, falling back to GitHub API: {exc.stderr.strip()}", - file=sys.stderr, - ) - return None - - return versions_from_tags(output.splitlines()) - - -def fetch_versions_with_requests() -> list[str]: - url = f"https://api.github.com/repos/{REPO}/releases" - page = 1 - releases = [] - while True: - resp = requests.get( - url, - params={"per_page": 100, "page": page}, - headers={"User-Agent": USER_AGENT}, - timeout=30, - ) - resp.raise_for_status() - data = resp.json() - if not data: - break - releases.extend(data) - page += 1 - - return versions_from_releases(releases) - - -def fetch_versions() -> list[str]: - """Fetch all TidesDB release versions from GitHub, sorted ascending.""" - versions = fetch_versions_with_gh() - if versions is not None: - return versions - return fetch_versions_with_requests() - - -def fetch_crate_api(name: str) -> dict | None: - resp = requests.get( - f"{CRATES_IO}/{name}", - headers={"User-Agent": USER_AGENT}, - timeout=30, - ) - if resp.status_code == 404: - return None - resp.raise_for_status() - return resp.json() - - -def latest_published_version(crate: dict) -> str | None: - versions = [ - version["num"] - for version in crate.get("versions", []) - if not version.get("yanked", False) - ] - if not versions: - return None - return sorted(versions, key=version_key)[-1] - - -def bump_patch(version: str) -> str: - major, minor, patch = version_key(version) - return f"{major}.{minor}.{patch + 1}" - - -def fetch_published_crate_bytes(name: str, version: str) -> bytes: - url = f"{CRATES_IO}/{name}/{version}/download" - resp = requests.get(url, headers={"User-Agent": USER_AGENT}, timeout=60) - resp.raise_for_status() - return resp.content - - -def read_member_bytes(crate_bytes: bytes, member_name: str) -> bytes | None: - with tarfile.open(fileobj=io.BytesIO(crate_bytes), mode="r:gz") as archive: - for member in archive.getmembers(): - if member.name.endswith(f"/{member_name}"): - file = archive.extractfile(member) - return file.read() if file else None - return None - - -def expected_manifest(version: str, package_version: str) -> str: - crate_name = source_crate_name(version) - cargo = tomlkit.document() - package = tomlkit.table() - package["name"] = crate_name - package["version"] = package_version - package["edition"] = "2024" - package["authors"] = ["TidesDB "] - package["description"] = f"Vendored TidesDB C source archive for v{version}" - package["license"] = "MPL-2.0" - package["repository"] = "https://github.com/tidesdb/tidesdb-rs" - package["homepage"] = "https://tidesdb.com" - package["readme"] = "README.md" - package["keywords"] = ["database", "key-value", "tidesdb", "source"] - package["categories"] = ["database", "external-ffi-bindings"] - package["include"] = ["src/**", "vendor/**", "Cargo.toml", "README.md", "LICENSE"] - cargo["package"] = package - cargo["lib"] = {"path": "src/lib.rs"} - return tomlkit.dumps(cargo) - - -def expected_lib_rs(version: str) -> str: - archive_name = source_archive_name(version) - return ( - "use std::path::{Path, PathBuf};\n\n" - f"pub const TIDESDB_VERSION: &str = \"{version}\";\n" - f"pub const ARCHIVE_NAME: &str = \"{archive_name}\";\n\n" - "pub fn archive_path() -> PathBuf {\n" - " Path::new(env!(\"CARGO_MANIFEST_DIR\"))\n" - " .join(\"vendor\")\n" - " .join(ARCHIVE_NAME)\n" - "}\n" - ) - - -def comparable_manifest(content: bytes, package_version: str) -> dict: - data = tomlkit.loads(content.decode("utf-8")) - package = to_plain(data["package"]) - package.pop("version", None) - package["version"] = package_version - return { - "package": package, - "lib": to_plain(data.get("lib", {})), - } - - -def to_plain(value): - if isinstance(value, dict): - return {str(key): to_plain(item) for key, item in value.items()} - if isinstance(value, list): - return [to_plain(item) for item in value] - return value.unwrap() if hasattr(value, "unwrap") else value - - -def archive_info(data: bytes) -> ArchiveInfo: - return ArchiveInfo( - data=data, - size=len(data), - sha256=hashlib.sha256(data).hexdigest(), - ) - - -def compare_published_crate( - version: str, - published_version: str, - published_bytes: bytes, - source_archive: ArchiveInfo, -) -> list[str]: - reasons = [] - - manifest = ( - read_member_bytes(published_bytes, "Cargo.toml.orig") - or read_member_bytes(published_bytes, "Cargo.toml") - ) - if manifest is None: - reasons.append("published crate is missing Cargo.toml") - else: - expected = comparable_manifest( - expected_manifest(version, published_version).encode("utf-8"), - published_version, - ) - actual = comparable_manifest(manifest, published_version) - if actual != expected: - reasons.append("package metadata differs") - - lib_rs = read_member_bytes(published_bytes, "src/lib.rs") - if lib_rs is None: - reasons.append("published crate is missing src/lib.rs") - elif lib_rs.decode("utf-8") != expected_lib_rs(version): - reasons.append("src/lib.rs metadata differs") - - archive = read_member_bytes(published_bytes, f"vendor/{source_archive_name(version)}") - if archive is None: - reasons.append("published crate is missing vendored archive") - else: - published_archive = archive_info(archive) - if published_archive.size != source_archive.size: - reasons.append( - f"vendored archive size differs ({published_archive.size} != {source_archive.size})" - ) - if published_archive.sha256 != source_archive.sha256: - reasons.append( - "vendored archive sha256 differs " - f"({published_archive.sha256} != {source_archive.sha256})" - ) - - return reasons - - -def fetch_crates_state(versions: Iterable[str]) -> dict[str, SourceCrateState]: - state = {} - for version in versions: - name = source_crate_name(version) - try: - crate = fetch_crate_api(name) - if crate is None: - state[version] = SourceCrateState( - status="missing", - publish_version=SOURCE_CRATE_INITIAL_VERSION, - reasons=["crate is not published"], - ) - continue - - published_version = latest_published_version(crate) - if published_version is None: - state[version] = SourceCrateState( - status="missing", - publish_version=SOURCE_CRATE_INITIAL_VERSION, - reasons=["crate has no non-yanked published versions"], - ) - continue - - source_archive = download_archive_info(version) - published_bytes = fetch_published_crate_bytes(name, published_version) - reasons = compare_published_crate( - version, - published_version, - published_bytes, - source_archive, - ) - if reasons: - state[version] = SourceCrateState( - status="update", - published_version=published_version, - publish_version=bump_patch(published_version), - reasons=reasons, - ) - else: - state[version] = SourceCrateState( - status="current", - published_version=published_version, - publish_version=published_version, - ) - except requests.HTTPError as exc: - print(f"warning: failed to query crates.io for {name}: {exc}", file=sys.stderr) - state[version] = SourceCrateState( - status="unknown", - reasons=[f"crates.io query failed: {exc}"], - ) - return state - - -def is_version(value: str) -> bool: - parts = value.split(".") - if len(parts) != 3: - return False - return all(part.isdigit() for part in parts) - - -def version_key(version: str) -> tuple[int, int, int]: - major, minor, patch = version.split(".") - return int(major), int(minor), int(patch) - - -def version_to_feature(version: str) -> str: - return "v" + version.replace(".", "_") - - -def feature_to_version(feature: str) -> str: - return feature[1:].replace("_", ".") - - -def source_crate_name(version: str) -> str: - return "tidesdb-src-v" + version.replace(".", "-") - - -def source_crate_rust_name(version: str) -> str: - return source_crate_name(version).replace("-", "_") - - -def source_archive_name(version: str) -> str: - return f"tidesdb-{version}.tar.gz" - - -def parse_versions(value: str, current: list[str], all_versions: list[str]) -> list[str]: - if value == "current": - return current - if value == "all": - return all_versions - - selected = [] - for raw in value.split(","): - version = raw.strip().removeprefix("v").replace("_", ".") - if not version: - continue - if not is_version(version): - raise SystemExit(f"invalid version: {raw}") - selected.append(version) - return sorted(set(selected), key=version_key) - - -def parse_publish_versions(value: str, selected: list[str], needed: list[str], missing: list[str]) -> list[str]: - if value == "none": - return [] - if value == "needed": - return needed - if value == "missing": - return missing - if value == "selected": - return [version for version in selected if version in needed] - - versions = parse_versions(value, current=selected, all_versions=selected) - unselected = sorted(set(versions) - set(selected), key=version_key) - if unselected: - raise SystemExit( - "--publish-versions must be a subset of --versions: " - + ", ".join(unselected) - ) - not_needed = sorted(set(versions) - set(needed), key=version_key) - if not_needed: - raise SystemExit( - "selected source crates do not need publish/update: " - + ", ".join(not_needed) - ) - return versions - - -def load_cargo_toml() -> tomlkit.TOMLDocument: - with CARGO_TOML.open("r") as f: - return tomlkit.load(f) - - -def save_cargo_toml(data: tomlkit.TOMLDocument): - with CARGO_TOML.open("w") as f: - tomlkit.dump(data, f) - - -def current_version_features(cargo: dict) -> set[str]: - return { - key - for key in cargo.get("features", {}) - if re.match(r"^v\d+_\d+_\d+$", key) - } - - -def current_versions(cargo: dict) -> list[str]: - return sorted( - [feature_to_version(feature) for feature in current_version_features(cargo)], - key=version_key, - ) - - -def current_default_feature(cargo: dict) -> str | None: - for feature in cargo.get("features", {}).get("default", []): - if re.match(r"^v\d+_\d+_\d+$", feature): - return feature - return None - - -def source_crate_requirement(version: str, state: SourceCrateState | None) -> str: - package_version = ( - state.publish_version - if state and state.publish_version - else SOURCE_CRATE_INITIAL_VERSION - ) - major, minor, _ = version_key(package_version) - return f"{major}.{minor}" - - -def dependency_value(version: str, state: SourceCrateState | None): - value = tomlkit.inline_table() - value["version"] = source_crate_requirement(version, state) - value["optional"] = True - return value - - -def update_cargo_toml( - cargo: tomlkit.TOMLDocument, - versions: list[str], - default_version: str, - crates_state: dict[str, SourceCrateState], -): - build_deps = cargo.setdefault("build-dependencies", tomlkit.table()) - if "ureq" in build_deps: - del build_deps["ureq"] - build_deps["cfg-if"] = "1" - - for key in list(build_deps.keys()): - if re.match(r"^tidesdb-src-v\d+-\d+-\d+$", key): - del build_deps[key] - - deps = cargo.setdefault("dependencies", tomlkit.table()) - for key in list(deps.keys()): - if re.match(r"^tidesdb-src-v\d+-\d+-\d+$", key): - del deps[key] - - for version in versions: - build_deps[source_crate_name(version)] = dependency_value(version, crates_state.get(version)) - - features = cargo.setdefault("features", tomlkit.table()) - for key in list(features.keys()): - if re.match(r"^v\d+_\d+_\d+$", key): - del features[key] - - new_default = [ - feature - for feature in features.get("default", []) - if not re.match(r"^v\d+_\d+_\d+$", feature) - ] - new_default.append(version_to_feature(default_version)) - features["default"] = new_default - - for version in versions: - features[version_to_feature(version)] = [f"dep:{source_crate_name(version)}"] - - -def render_cargo_toml( - cargo: tomlkit.TOMLDocument, - versions: list[str], - default_version: str, - crates_state: dict[str, SourceCrateState], -) -> str: - rendered = copy.deepcopy(cargo) - update_cargo_toml(rendered, versions, default_version, crates_state) - return tomlkit.dumps(rendered) - - -def source_archive_function(versions: list[str]) -> str: - branches = [] - for version in reversed(versions): - feature = version_to_feature(version) - crate = source_crate_rust_name(version) - keyword = "if" if not branches else "else if" - branches.append( - f' {keyword} #[cfg(feature = "{feature}")] {{\n' - " fn source_archive() -> PathBuf {\n" - f" {crate}::archive_path()\n" - " }\n" - " }" - ) - branch_text = "\n".join(branches) - return ( - "cfg_if::cfg_if! {\n" - f"{branch_text}\n" - " else {\n" - " fn source_archive() -> PathBuf {\n" - " panic!(\"no tidesdb source crate is enabled; enable a vX_Y_Z feature\")\n" - " }\n" - " }\n" - "}\n" - ) - - -def extraction_functions(versions: list[str]) -> str: - return ( - "// BEGIN GENERATED TIDESDB SOURCE ARCHIVES\n" - f"{source_archive_function(versions)}\n" - "// END GENERATED TIDESDB SOURCE ARCHIVES\n\n" - "fn extract_archive(version: &str, out_dir: &str) -> PathBuf {\n" - " let archive_path = source_archive();\n" - " let extract_dir = PathBuf::from(out_dir).join(\"tidesdb-source\");\n" - " let src_dir = extract_dir.join(format!(\"tidesdb-{version}\"));\n\n" - " if src_dir.exists() {\n" - " return src_dir;\n" - " }\n\n" - " let tarball = std::fs::File::open(&archive_path).unwrap_or_else(|e| {\n" - " panic!(\n" - " \"failed to open TidesDB v{version} source archive {}: {e}\",\n" - " archive_path.display()\n" - " )\n" - " });\n" - " let decoder = flate2::read::GzDecoder::new(tarball);\n" - " let mut archive = tar::Archive::new(decoder);\n" - " std::fs::create_dir_all(&extract_dir).expect(\"Failed to create extract directory\");\n" - " archive\n" - " .unpack(&extract_dir)\n" - " .expect(\"Failed to extract TidesDB source archive\");\n\n" - " src_dir\n" - "}\n" - ) - - -def render_build_rs(content: str, versions: list[str], default_version: str) -> str: - content = re.sub( - r'None => "[\d.]+"\.to_string\(\)', - f'None => "{default_version}".to_string()', - content, - ) - - generated = extraction_functions(versions) - if "BEGIN GENERATED TIDESDB SOURCE ARCHIVES" in content: - content = re.sub( - r"// BEGIN GENERATED TIDESDB SOURCE ARCHIVES\n.*?// END GENERATED TIDESDB SOURCE ARCHIVES\n\n" - r"fn extract_archive\(version: &str, out_dir: &str\) -> PathBuf \{\n.*?\n\}\n", - generated, - content, - flags=re.S, - ) - else: - content = re.sub( - r"fn download_and_extract\(version: &str, out_dir: &str\) -> PathBuf \{\n.*?\n\}\n\n", - generated + "\n", - content, - flags=re.S, - ) - - content = content.replace( - "let src_dir = download_and_extract(version, &out_dir);", - "let src_dir = extract_archive(version, &out_dir);", - ) - - return content - - -def update_build_rs(versions: list[str], default_version: str): - BUILD_RS.write_text(render_build_rs(BUILD_RS.read_text(), versions, default_version)) - - -def files_to_modify_for_plan( - cargo: tomlkit.TOMLDocument, - selected_versions: list[str], - default_version: str, - crates_state: dict[str, SourceCrateState], -) -> list[Path]: - files = [] - if render_cargo_toml(cargo, selected_versions, default_version, crates_state) != CARGO_TOML.read_text(): - files.append(CARGO_TOML) - if render_build_rs(BUILD_RS.read_text(), selected_versions, default_version) != BUILD_RS.read_text(): - files.append(BUILD_RS) - if BUILD_AND_TEST_YML.exists(): - current_default = current_default_feature(cargo) - if current_default != version_to_feature(default_version): - files.append(BUILD_AND_TEST_YML) - return files - - -def update_ci_default(version: str): - if not BUILD_AND_TEST_YML.exists(): - return - - content = BUILD_AND_TEST_YML.read_text() - content = re.sub( - r"(repository: tidesdb/tidesdb\s+ref: )v[\d.]+", - rf"\g<1>v{version}", - content, - ) - BUILD_AND_TEST_YML.write_text(content) - - -def download_archive(version: str, destination: Path): - info = download_archive_info(version) - destination.write_bytes(info.data) - - -def download_archive_info(version: str) -> ArchiveInfo: - url = f"https://github.com/{REPO}/archive/refs/tags/v{version}.tar.gz" - with requests.get(url, headers={"User-Agent": USER_AGENT}, stream=True, timeout=60) as resp: - resp.raise_for_status() - data = b"".join(chunk for chunk in resp.iter_content(chunk_size=1024 * 1024) if chunk) - return archive_info(data) - - -def write_source_crate( - version: str, - package_version: str, - root: Path, - *, - include_archive: bool = True, -): - crate_name = source_crate_name(version) - crate_dir = root / crate_name - if crate_dir.exists(): - shutil.rmtree(crate_dir) - vendor_dir = crate_dir / "vendor" - src_dir = crate_dir / "src" - vendor_dir.mkdir(parents=True) - src_dir.mkdir(parents=True) - - archive_name = source_archive_name(version) - if include_archive: - download_archive(version, vendor_dir / archive_name) - - (crate_dir / "Cargo.toml").write_text(expected_manifest(version, package_version)) - - (crate_dir / "README.md").write_text( - f"# {crate_name}\n\n" - f"This crate packages the TidesDB C source archive for `v{version}`.\n\n" - "It is intended as a build-dependency of the `tidesdb` Rust bindings.\n" - ) - - if LICENSE.exists(): - shutil.copyfile(LICENSE, crate_dir / "LICENSE") - - (src_dir / "lib.rs").write_text(expected_lib_rs(version)) - - return crate_dir - - -def dependency_version(cargo: tomlkit.TOMLDocument, version: str) -> str: - crate_name = source_crate_name(version) - dependency = cargo.get("build-dependencies", {}).get(crate_name) - if isinstance(dependency, str): - requirement = dependency - elif isinstance(dependency, dict): - requirement = dependency.get("version", SOURCE_CRATE_INITIAL_VERSION) - else: - requirement = SOURCE_CRATE_INITIAL_VERSION - - match = re.fullmatch(r"(\d+)\.(\d+)(?:\.(\d+))?", str(requirement)) - if not match: - return SOURCE_CRATE_INITIAL_VERSION - - major, minor, patch = match.groups() - return f"{major}.{minor}.{patch or 0}" - - -def parse_local_patch_archive_versions( - value: str, - *, - selected_versions: list[str], - default_version: str, -) -> set[str]: - if value == "all": - return set(selected_versions) - if value == "current": - return set(selected_versions) - if value == "default": - return {default_version} - if value == "none": - return set() - return set(parse_versions(value, current=selected_versions, all_versions=selected_versions)) - - -def write_cargo_patch_config(config_path: Path, crate_dirs: dict[str, Path]): - if config_path.exists(): - config = tomlkit.parse(config_path.read_text()) - else: - config = tomlkit.document() - - patch = config.setdefault("patch", tomlkit.table()) - crates_io = patch.setdefault("crates-io", tomlkit.table()) - - for key in list(crates_io.keys()): - if re.match(r"^tidesdb-src-v\d+-\d+-\d+$", key): - del crates_io[key] - - for version, crate_dir in sorted(crate_dirs.items(), key=lambda item: version_key(item[0])): - value = tomlkit.inline_table() - value["path"] = crate_dir.resolve().as_posix() - crates_io[source_crate_name(version)] = value - - config_path.parent.mkdir(parents=True, exist_ok=True) - config_path.write_text(tomlkit.dumps(config)) - - -def prepare_local_patches( - cargo: tomlkit.TOMLDocument, - versions: list[str], - work_root: Path, - config_path: Path | None, - archive_versions: set[str], -): - work_root.mkdir(parents=True, exist_ok=True) - crate_dirs = {} - - for version in versions: - package_version = dependency_version(cargo, version) - include_archive = version in archive_versions - crate_dir = write_source_crate( - version, - package_version, - work_root, - include_archive=include_archive, - ) - crate_dirs[version] = crate_dir - if include_archive: - print(f"Prepared {source_crate_name(version)}@{package_version} with vendored archive") - else: - print(f"Prepared {source_crate_name(version)}@{package_version} for dependency patching") - - if config_path: - write_cargo_patch_config(config_path, crate_dirs) - print(f"Wrote Cargo patch config to {config_path}") - - -def cargo_publish(crate_dir: Path, dry_run: bool): - cmd = ["cargo", "publish", "--manifest-path", str(crate_dir / "Cargo.toml")] - if dry_run: - cmd.append("--dry-run") - subprocess.run(cmd, check=True) - - -def summarize( - selected_versions: list[str], - default_version: str, - existing_features: set[str], - crates_state: dict[str, SourceCrateState], - publish_versions: list[str], - files_to_modify: list[Path], - publish: bool, - source_crates_only: bool, -): - selected_features = {version_to_feature(v) for v in selected_versions} - added = selected_features - existing_features - removed = existing_features - selected_features - current_crates = [ - f"{source_crate_name(v)}@{crates_state[v].published_version}" - for v in selected_versions - if crates_state.get(v) and crates_state[v].status == "current" - ] - missing_crates = [ - f"{source_crate_name(v)} -> {crates_state[v].publish_version}" - for v in selected_versions - if crates_state.get(v) and crates_state[v].status == "missing" - ] - unknown_crates = [ - source_crate_name(v) - for v in selected_versions - if crates_state.get(v) and crates_state[v].status == "unknown" - ] - update_crates = [ - f"{source_crate_name(v)} {crates_state[v].published_version} -> {crates_state[v].publish_version}" - for v in selected_versions - if crates_state.get(v) and crates_state[v].status == "update" - ] - - print("\nPlan:") - print(f" Version features: {', '.join(version_to_feature(v) for v in selected_versions)}") - print(f" Default: {version_to_feature(default_version)} ({default_version})") - if source_crates_only: - print(" Root changes: skipped") - elif added: - print(f" Add features: {', '.join(sorted(added))}") - if not source_crates_only and removed: - print(f" Remove features: {', '.join(sorted(removed))}") - if current_crates: - print(f" Current crates: {', '.join(current_crates)}") - if missing_crates: - print(f" Missing crates: {', '.join(missing_crates)}") - if unknown_crates: - print(f" Unknown crates: {', '.join(unknown_crates)}") - for version in selected_versions: - state = crates_state.get(version) - if state and state.status == "unknown": - print(f" {source_crate_name(version)}: {'; '.join(state.reasons)}") - if update_crates: - print(f" Update crates: {', '.join(update_crates)}") - for version in selected_versions: - state = crates_state.get(version) - if state and state.status == "update": - print(f" {source_crate_name(version)}: {'; '.join(state.reasons)}") - if publish_versions: - action = "Publish crates" if publish else "Check crates" - print( - f" {action}: " - + ", ".join( - f"{source_crate_name(v)}@{crates_state[v].publish_version}" - for v in publish_versions - ) - ) - else: - print(" Publish/check: none") - if source_crates_only: - files = "none (--source-crates-only)" - else: - files = ", ".join(str(path) for path in files_to_modify) if files_to_modify else "none" - print(f" Files to modify: {files}") - - -def choose_interactively( - all_versions: list[str], - current: list[str], - current_default: str | None, - crates_state: dict[str, SourceCrateState], -) -> tuple[list[str], str, list[str]]: - choices = [] - for version in all_versions: - feature = version_to_feature(version) - state = crates_state[version] - if state.status == "current": - status = f"published {state.published_version}" - elif state.status == "update": - status = f"update {state.published_version} -> {state.publish_version}" - elif state.status == "unknown": - status = "unknown; registry check failed" - else: - status = f"missing -> {state.publish_version}" - name = f"{feature} ({version}, {status})" - if feature == current_default: - name += " [current default]" - choices.append( - { - "name": name, - "value": version, - "enabled": version in current, - } - ) - - selected_versions = inquirer.checkbox( - message="Select version features:", - choices=choices, - instruction="(Space to toggle, Enter to confirm)", - ).execute() - - if not selected_versions: - raise SystemExit("No versions selected, aborting.") - - selected_versions = sorted(selected_versions, key=version_key) - default_current = feature_to_version(current_default) if current_default else None - default_default = default_current if default_current in selected_versions else selected_versions[-1] - - default_version = inquirer.select( - message="Select default version:", - choices=[ - {"name": f"{version_to_feature(v)} ({v})", "value": v} - for v in selected_versions - ], - default=default_default, - ).execute() - - needed = [version for version in selected_versions if crates_state[version].needs_publish] - publish_versions = [] - if needed: - publish_versions = inquirer.checkbox( - message="Select source crates to publish/update:", - choices=[ - { - "name": ( - f"{source_crate_name(version)} " - f"({crates_state[version].status}, " - f"{crates_state[version].published_version or 'none'}" - f" -> {crates_state[version].publish_version})" - ), - "value": version, - "enabled": True, - } - for version in needed - ], - instruction="(Space to toggle, Enter to confirm)", - ).execute() - publish_versions = sorted(publish_versions, key=version_key) - - return selected_versions, default_version, publish_versions - - -def main(): - if hasattr(sys.stdout, "reconfigure"): - sys.stdout.reconfigure(line_buffering=True) - - parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter, - ) - parser.add_argument("--dry-run", action="store_true", help="Show the plan without writing files or publishing crates") - parser.add_argument("--yes", action="store_true", help="Apply the plan without an interactive final confirmation") - parser.add_argument( - "--versions", - help="Comma-separated versions, 'current', or 'all'. Without this, use the interactive picker.", - ) - parser.add_argument("--default-version", help="Default TidesDB version feature to enable") - parser.add_argument( - "--publish-versions", - default=None, - help="Comma-separated versions to prepare, 'needed', 'missing', 'selected', or 'none'. Defaults to needed in non-interactive mode.", - ) - parser.add_argument("--skip-publish", action="store_true", help="Update files and prepare no crates") - parser.add_argument("--publish", action="store_true", help="Actually upload prepared crates to crates.io") - parser.add_argument("--publish-dry-run", action="store_true", help="Force cargo publish --dry-run for prepared crates") - parser.add_argument( - "--prepare-local-patches", - type=Path, - help="Prepare local source crates for Cargo dependency patching and exit", - ) - parser.add_argument( - "--write-cargo-patch-config", - type=Path, - help="Write [patch.crates-io] entries pointing at prepared local source crates", - ) - parser.add_argument( - "--local-patch-archive-versions", - default="default", - help="Versions whose local patch crates should include vendored archives: default, all, current, none, or a comma-separated list", - ) - parser.add_argument( - "--source-crates-only", - action="store_true", - help="Only prepare/check/publish source crates; do not update Cargo.toml, build.rs, or CI", - ) - parser.add_argument("--keep-work-dir", action="store_true", help="Keep prepared source crates instead of cleaning them up") - parser.add_argument("--work-dir", type=Path, help="Directory for prepared source crates") - args = parser.parse_args() - - cargo = load_cargo_toml() - current = current_versions(cargo) - existing_features = current_version_features(cargo) - current_default = current_default_feature(cargo) - default_current = feature_to_version(current_default) if current_default else None - - if args.prepare_local_patches: - selected_versions = parse_versions( - args.versions or "current", - current=current, - all_versions=current, - ) - default_version = args.default_version or ( - default_current if default_current in selected_versions else selected_versions[-1] - ) - if default_version not in selected_versions: - raise SystemExit("--default-version must be one of the selected versions") - archive_versions = parse_local_patch_archive_versions( - args.local_patch_archive_versions, - selected_versions=selected_versions, - default_version=default_version, - ) - prepare_local_patches( - cargo, - selected_versions, - args.prepare_local_patches, - args.write_cargo_patch_config, - archive_versions, - ) - return - - if args.write_cargo_patch_config: - raise SystemExit("--write-cargo-patch-config requires --prepare-local-patches") - - print(f"Fetching releases from {REPO}...") - all_versions = fetch_versions() - print(f"Found {len(all_versions)} releases.") - - if args.versions: - selected_versions = parse_versions(args.versions, current=current, all_versions=all_versions) - unknown = set(selected_versions) - set(all_versions) - if unknown: - raise SystemExit(f"selected versions are not TidesDB releases: {', '.join(sorted(unknown))}") - print("Surveying crates.io source crates...") - crates_state = fetch_crates_state(selected_versions) - default_version = args.default_version or ( - feature_to_version(current_default) - if current_default and feature_to_version(current_default) in selected_versions - else selected_versions[-1] - ) - if default_version not in selected_versions: - raise SystemExit("--default-version must be one of --versions") - needed = [version for version in selected_versions if crates_state[version].needs_publish] - missing = [version for version in selected_versions if crates_state[version].status == "missing"] - if args.skip_publish: - publish_versions = [] - elif args.publish_versions is None: - publish_versions = needed - else: - publish_versions = parse_publish_versions( - args.publish_versions, - selected_versions, - needed, - missing, - ) - else: - print("Surveying crates.io source crates...") - crates_state = fetch_crates_state(all_versions) - selected_versions, default_version, publish_versions = choose_interactively( - all_versions, - current, - current_default, - crates_state, - ) - if args.skip_publish: - publish_versions = [] - elif args.publish_versions is not None: - needed = [version for version in selected_versions if crates_state[version].needs_publish] - missing = [version for version in selected_versions if crates_state[version].status == "missing"] - publish_versions = parse_publish_versions( - args.publish_versions, - selected_versions, - needed, - missing, - ) - - files_to_modify = [] - if not args.source_crates_only: - files_to_modify = files_to_modify_for_plan( - cargo, - selected_versions, - default_version, - crates_state, - ) - - summarize( - selected_versions, - default_version, - existing_features, - crates_state, - publish_versions, - files_to_modify, - args.publish and not args.publish_dry_run, - args.source_crates_only, - ) - - blocking_versions = [ - version for version in selected_versions if crates_state[version].is_blocking - ] - if blocking_versions: - print("\nCannot continue while source crate state is unknown:") - for version in blocking_versions: - state = crates_state[version] - print(f" {source_crate_name(version)}: {'; '.join(state.reasons)}") - raise SystemExit(1) - - if args.dry_run: - print("\n--dry-run: no files modified and no crates published.") - return - - if not args.yes: - confirmed = inquirer.confirm(message="Apply this plan?", default=False).execute() - if not confirmed: - print("Aborted.") - return - - if not args.source_crates_only: - update_cargo_toml(cargo, selected_versions, default_version, crates_state) - save_cargo_toml(cargo) - update_build_rs(selected_versions, default_version) - update_ci_default(default_version) - - if not publish_versions: - print("Done.") - return - - if args.work_dir: - work_root = args.work_dir - work_root.mkdir(parents=True, exist_ok=True) - cleanup = False - else: - tempdir = tempfile.TemporaryDirectory(prefix="tidesdb-src-crates-") - work_root = Path(tempdir.name) - cleanup = not args.keep_work_dir - - try: - for version in publish_versions: - state = crates_state[version] - crate_dir = write_source_crate(version, state.publish_version, work_root) - publish = args.publish and not args.publish_dry_run - action = "Publishing" if publish else "Checking" - print(f"{action} {source_crate_name(version)}@{state.publish_version} from {crate_dir}...", flush=True) - cargo_publish(crate_dir, dry_run=not publish) - finally: - if cleanup and "tempdir" in locals(): - tempdir.cleanup() - else: - print(f"Kept prepared crates in {work_root}") - - print("Done.") - - -if __name__ == "__main__": - main() diff --git a/src/config.rs b/src/config.rs index ba43474..49ed01c 100644 --- a/src/config.rs +++ b/src/config.rs @@ -5,430 +5,188 @@ //! Configuration types for TidesDB. -use crate::error::{Result, check_result}; +use crate::error::{Error, Result}; use crate::ffi; use std::ffi::CString; use std::path::Path; -/// Compression algorithm for column families. +/// Compression algorithm, used as a stage in a column family's encoding pipeline. +/// +/// Every algorithm is always defined, but a backend is only usable when the C library +/// was built with it (the `snappy`, `lz4` and `zstd` Cargo features). Check with +/// [`CompressionAlgorithm::is_available`]. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] #[repr(i32)] pub enum CompressionAlgorithm { /// No compression #[default] - None = ffi::NO_COMPRESSION, + None = ffi::TDB_COMPRESS_NONE, /// Snappy compression - Snappy = ffi::SNAPPY_COMPRESSION, - /// LZ4 compression (default in C API) - Lz4 = ffi::LZ4_COMPRESSION, + Snappy = ffi::TDB_COMPRESS_SNAPPY, + /// LZ4 compression + Lz4 = ffi::TDB_COMPRESS_LZ4, /// Zstandard compression - Zstd = ffi::ZSTD_COMPRESSION, + Zstd = ffi::TDB_COMPRESS_ZSTD, /// LZ4 fast compression - Lz4Fast = ffi::LZ4_FAST_COMPRESSION, + Lz4Fast = ffi::TDB_COMPRESS_LZ4_FAST, } -/// Sync mode for durability. +impl CompressionAlgorithm { + /// Whether this build of the C library can compress and decompress with this algorithm. + /// [`CompressionAlgorithm::None`] is always available. + pub fn is_available(self) -> bool { + unsafe { ffi::tidesdb_compression_available(self as i32) != 0 } + } + + /// The algorithm with the given encoding id, if it names one. + pub fn from_id(id: u8) -> Option { + match id as i32 { + ffi::TDB_COMPRESS_NONE => Some(CompressionAlgorithm::None), + ffi::TDB_COMPRESS_SNAPPY => Some(CompressionAlgorithm::Snappy), + ffi::TDB_COMPRESS_LZ4 => Some(CompressionAlgorithm::Lz4), + ffi::TDB_COMPRESS_ZSTD => Some(CompressionAlgorithm::Zstd), + ffi::TDB_COMPRESS_LZ4_FAST => Some(CompressionAlgorithm::Lz4Fast), + _ => None, + } + } +} + +/// Durability mode for the write-ahead log. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] #[repr(i32)] pub enum SyncMode { - /// No sync (fastest, least durable) + /// Nothing at commit; fastest, a crash of the process or the machine may lose commits #[default] None = ffi::TDB_SYNC_NONE, - /// Full sync on every write (slowest, most durable) + /// fsync every commit; slowest, no commit loss on crash Full = ffi::TDB_SYNC_FULL, - /// Sync at intervals + /// fsync on a background interval; only a machine crash can lose the last interval Interval = ffi::TDB_SYNC_INTERVAL, } -/// Logging level. +impl SyncMode { + fn from_c(mode: i32) -> Self { + match mode { + ffi::TDB_SYNC_FULL => SyncMode::Full, + ffi::TDB_SYNC_INTERVAL => SyncMode::Interval, + _ => SyncMode::None, + } + } +} + +/// Logging threshold. A message is emitted when its severity meets or exceeds the level. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] #[repr(i32)] pub enum LogLevel { - /// Debug level - Debug = ffi::TDB_LOG_DEBUG, - /// Info level + /// No logging + None = ffi::TDB_LOG_NONE, + /// Highly detailed messages for technical debugging + Trace = ffi::TDB_LOG_TRACE, + /// Engine status and operations #[default] Info = ffi::TDB_LOG_INFO, - /// Warning level + /// Non-imminent errors that require awareness Warn = ffi::TDB_LOG_WARN, - /// Error level + /// An operation failed Error = ffi::TDB_LOG_ERROR, - /// Fatal level - Fatal = ffi::TDB_LOG_FATAL, - /// No logging - None = ffi::TDB_LOG_NONE, } -/// Transaction isolation level. +impl LogLevel { + fn from_c(level: i32) -> Self { + match level { + ffi::TDB_LOG_NONE => LogLevel::None, + ffi::TDB_LOG_TRACE => LogLevel::Trace, + ffi::TDB_LOG_WARN => LogLevel::Warn, + ffi::TDB_LOG_ERROR => LogLevel::Error, + _ => LogLevel::Info, + } + } +} + +/// Transaction isolation level, weakest to strongest. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] #[repr(i32)] pub enum IsolationLevel { - /// Read uncommitted - sees all data including uncommitted changes + /// Every version is visible, including sequences still in progress ReadUncommitted = ffi::TDB_ISOLATION_READ_UNCOMMITTED, - /// Read committed - sees only committed data (default) + /// The newest committed version, re-read on every operation (default) #[default] ReadCommitted = ffi::TDB_ISOLATION_READ_COMMITTED, - /// Repeatable read - consistent snapshot, phantom reads possible + /// Frozen snapshot; commit validates that every key read is unchanged and that no key + /// appeared inside a scanned range RepeatableRead = ffi::TDB_ISOLATION_REPEATABLE_READ, - /// Snapshot - write-write conflict detection + /// Frozen snapshot; commit claims each written key first-committer-wins Snapshot = ffi::TDB_ISOLATION_SNAPSHOT, - /// Serializable - full read-write conflict detection (SSI) + /// Both of the checks above Serializable = ffi::TDB_ISOLATION_SERIALIZABLE, } -/// Object store behavior configuration. -/// -/// Controls caching, upload/download parallelism, multipart thresholds, -/// WAL replication, and replica mode for object store deployments. -#[derive(Debug, Clone)] -pub struct ObjectStoreConfig { - /// Local directory for cached SSTable files (None = use db_path) - pub local_cache_path: Option, - /// Maximum local cache size in bytes (0 = unlimited) - pub local_cache_max_bytes: usize, - /// Cache downloaded files locally (default: true) - pub cache_on_read: bool, - /// Keep local copy after upload (default: true) - pub cache_on_write: bool, - /// Number of parallel upload threads (default: 4) - pub max_concurrent_uploads: i32, - /// Number of parallel download threads (default: 8) - pub max_concurrent_downloads: i32, - /// Use multipart upload above this size in bytes (default: 64MB) - pub multipart_threshold: usize, - /// Chunk size for multipart uploads in bytes (default: 8MB) - pub multipart_part_size: usize, - /// Upload MANIFEST after each compaction (default: true) - pub sync_manifest_to_object: bool, - /// Upload closed WAL segments for replication (default: true) - pub replicate_wal: bool, - /// false = background WAL upload (default), true = block flush until uploaded - pub wal_upload_sync: bool, - /// Sync active WAL when it grows by this many bytes (default: 1MB, 0 = off) - pub wal_sync_threshold_bytes: usize, - /// Upload WAL after every txn commit for RPO=0 replication (default: false) - pub wal_sync_on_commit: bool, - /// Enable read-only replica mode (default: false) - pub replica_mode: bool, - /// MANIFEST poll interval for replica sync in microseconds (default: 5s) - pub replica_sync_interval_us: u64, - /// Replay WAL from object store for near-real-time reads on replicas (default: true) - pub replica_replay_wal: bool, -} - -impl ObjectStoreConfig { - /// Create a new object store configuration with default values. - pub fn new() -> Self { - Self::default() - } - - /// Set the local cache path (None = use db_path). - pub fn local_cache_path(mut self, path: &str) -> Self { - self.local_cache_path = Some(path.to_string()); - self - } - - /// Set the maximum local cache size in bytes (0 = unlimited). - pub fn local_cache_max_bytes(mut self, size: usize) -> Self { - self.local_cache_max_bytes = size; - self - } - - /// Enable or disable caching downloaded files locally. - pub fn cache_on_read(mut self, enable: bool) -> Self { - self.cache_on_read = enable; - self - } - - /// Enable or disable keeping local copy after upload. - pub fn cache_on_write(mut self, enable: bool) -> Self { - self.cache_on_write = enable; - self - } - - /// Set the number of parallel upload threads. - pub fn max_concurrent_uploads(mut self, n: i32) -> Self { - self.max_concurrent_uploads = n; - self - } - - /// Set the number of parallel download threads. - pub fn max_concurrent_downloads(mut self, n: i32) -> Self { - self.max_concurrent_downloads = n; - self - } - - /// Set the multipart upload threshold in bytes. - pub fn multipart_threshold(mut self, size: usize) -> Self { - self.multipart_threshold = size; - self - } - - /// Set the multipart chunk size in bytes. - pub fn multipart_part_size(mut self, size: usize) -> Self { - self.multipart_part_size = size; - self - } - - /// Enable or disable uploading MANIFEST after each compaction. - pub fn sync_manifest_to_object(mut self, enable: bool) -> Self { - self.sync_manifest_to_object = enable; - self - } - - /// Enable or disable uploading closed WAL segments for replication. - pub fn replicate_wal(mut self, enable: bool) -> Self { - self.replicate_wal = enable; - self - } - - /// Set WAL upload sync mode (false = background, true = block flush until uploaded). - pub fn wal_upload_sync(mut self, enable: bool) -> Self { - self.wal_upload_sync = enable; - self - } - - /// Set the WAL sync threshold in bytes (0 = off). - pub fn wal_sync_threshold_bytes(mut self, size: usize) -> Self { - self.wal_sync_threshold_bytes = size; - self - } - - /// Enable or disable uploading WAL after every txn commit (RPO=0). - pub fn wal_sync_on_commit(mut self, enable: bool) -> Self { - self.wal_sync_on_commit = enable; - self - } - - /// Enable or disable read-only replica mode. - pub fn replica_mode(mut self, enable: bool) -> Self { - self.replica_mode = enable; - self - } - - /// Set the MANIFEST poll interval for replica sync in microseconds. - pub fn replica_sync_interval_us(mut self, interval: u64) -> Self { - self.replica_sync_interval_us = interval; - self - } - - /// Enable or disable WAL replay on replicas for near-real-time reads. - pub fn replica_replay_wal(mut self, enable: bool) -> Self { - self.replica_replay_wal = enable; - self - } - - /// Convert to C configuration struct. - /// Returns the C struct and an optional CString for the cache path that must stay alive. - pub(crate) fn to_c_config(&self) -> (ffi::tidesdb_objstore_config_t, Option) { - let (cache_path_ptr, cache_path_cstr) = match &self.local_cache_path { - Some(p) => { - let cs = CString::new(p.as_str()).unwrap_or_default(); - let ptr = cs.as_ptr(); - (ptr, Some(cs)) - } - None => (std::ptr::null(), None), - }; - - let config = ffi::tidesdb_objstore_config_t { - local_cache_path: cache_path_ptr, - local_cache_max_bytes: self.local_cache_max_bytes, - cache_on_read: if self.cache_on_read { 1 } else { 0 }, - cache_on_write: if self.cache_on_write { 1 } else { 0 }, - max_concurrent_uploads: self.max_concurrent_uploads, - max_concurrent_downloads: self.max_concurrent_downloads, - multipart_threshold: self.multipart_threshold, - multipart_part_size: self.multipart_part_size, - sync_manifest_to_object: if self.sync_manifest_to_object { 1 } else { 0 }, - replicate_wal: if self.replicate_wal { 1 } else { 0 }, - wal_upload_sync: if self.wal_upload_sync { 1 } else { 0 }, - wal_sync_threshold_bytes: self.wal_sync_threshold_bytes, - wal_sync_on_commit: if self.wal_sync_on_commit { 1 } else { 0 }, - replica_mode: if self.replica_mode { 1 } else { 0 }, - replica_sync_interval_us: self.replica_sync_interval_us, - replica_replay_wal: if self.replica_replay_wal { 1 } else { 0 }, - }; - - (config, cache_path_cstr) - } -} - -impl Default for ObjectStoreConfig { - fn default() -> Self { - let c = unsafe { ffi::tidesdb_objstore_default_config() }; - ObjectStoreConfig { - local_cache_path: None, - local_cache_max_bytes: c.local_cache_max_bytes, - cache_on_read: c.cache_on_read != 0, - cache_on_write: c.cache_on_write != 0, - max_concurrent_uploads: c.max_concurrent_uploads, - max_concurrent_downloads: c.max_concurrent_downloads, - multipart_threshold: c.multipart_threshold, - multipart_part_size: c.multipart_part_size, - sync_manifest_to_object: c.sync_manifest_to_object != 0, - replicate_wal: c.replicate_wal != 0, - wal_upload_sync: c.wal_upload_sync != 0, - wal_sync_threshold_bytes: c.wal_sync_threshold_bytes, - wal_sync_on_commit: c.wal_sync_on_commit != 0, - replica_mode: c.replica_mode != 0, - replica_sync_interval_us: c.replica_sync_interval_us, - replica_replay_wal: c.replica_replay_wal != 0, - } - } -} - -/// S3-compatible object store connector configuration. -/// -/// Used with [`Config::object_store_s3`] to back a database with an S3 bucket -/// (AWS S3, MinIO, etc.). Requires the `objectstore` Cargo feature and a TidesDB -/// C library built with `TIDESDB_WITH_S3=ON`. -/// -/// The all-default values are secure: TLS verification is on, no custom CA is -/// used, and the library's built-in multipart sizes apply (`0` = library default). -#[cfg(feature = "objectstore")] -#[derive(Debug, Clone, Default)] -pub struct S3Config { - /// S3 endpoint (e.g. `"s3.amazonaws.com"` or `"minio.local:9000"`). - pub endpoint: String, - /// Bucket name. - pub bucket: String, - /// Key prefix (e.g. `"production/db1/"`), or `None`. - pub prefix: Option, - /// AWS access key ID. - pub access_key: String, - /// AWS secret access key. - pub secret_key: String, - /// AWS region (e.g. `"us-east-1"`), or `None` for MinIO / the default. - pub region: Option, - /// `true` for HTTPS, `false` for HTTP. - pub use_ssl: bool, - /// `true` for path-style URLs (MinIO), `false` for virtual-hosted (AWS). - pub use_path_style: bool, - /// Custom CA bundle file path, or `None` for the system bundle. - pub tls_ca_path: Option, - /// `true` disables TLS peer+host verification (test only, insecure). - pub tls_insecure_skip_verify: bool, - /// Object size at/above which multipart upload is used; `0` = library default. - pub multipart_threshold: usize, - /// Multipart chunk size in bytes; `0` = library default. - pub multipart_part_size: usize, -} - -#[cfg(feature = "objectstore")] -impl S3Config { - /// Create a new S3 configuration with the required endpoint, bucket, and credentials. - pub fn new(endpoint: &str, bucket: &str, access_key: &str, secret_key: &str) -> Self { - S3Config { - endpoint: endpoint.to_string(), - bucket: bucket.to_string(), - access_key: access_key.to_string(), - secret_key: secret_key.to_string(), - ..Default::default() +impl IsolationLevel { + fn from_c(level: i32) -> Self { + match level { + ffi::TDB_ISOLATION_READ_UNCOMMITTED => IsolationLevel::ReadUncommitted, + ffi::TDB_ISOLATION_REPEATABLE_READ => IsolationLevel::RepeatableRead, + ffi::TDB_ISOLATION_SNAPSHOT => IsolationLevel::Snapshot, + ffi::TDB_ISOLATION_SERIALIZABLE => IsolationLevel::Serializable, + _ => IsolationLevel::ReadCommitted, } } - - /// Set the key prefix. - pub fn prefix(mut self, prefix: &str) -> Self { - self.prefix = Some(prefix.to_string()); - self - } - - /// Set the AWS region. - pub fn region(mut self, region: &str) -> Self { - self.region = Some(region.to_string()); - self - } - - /// Enable or disable HTTPS (TLS). - pub fn use_ssl(mut self, enable: bool) -> Self { - self.use_ssl = enable; - self - } - - /// Enable or disable path-style URLs (required for MinIO). - pub fn use_path_style(mut self, enable: bool) -> Self { - self.use_path_style = enable; - self - } - - /// Set a custom CA bundle file path. - pub fn tls_ca_path(mut self, path: &str) -> Self { - self.tls_ca_path = Some(path.to_string()); - self - } - - /// Disable TLS peer+host verification (test only, insecure). - pub fn tls_insecure_skip_verify(mut self, enable: bool) -> Self { - self.tls_insecure_skip_verify = enable; - self - } - - /// Set the multipart upload threshold in bytes (`0` = library default). - pub fn multipart_threshold(mut self, size: usize) -> Self { - self.multipart_threshold = size; - self - } - - /// Set the multipart chunk size in bytes (`0` = library default). - pub fn multipart_part_size(mut self, size: usize) -> Self { - self.multipart_part_size = size; - self - } } -/// Holds all C-side allocations that must outlive the `tidesdb_open()` call. +/// C configuration plus the owned data its pointers refer to; must outlive `tidesdb_open`. pub(crate) struct CConfigData { pub config: ffi::tidesdb_config_t, _db_path: CString, - _objstore_config: Option>, - _cache_path: Option, } /// Database configuration. +/// +/// The memtable, write-ahead log, value log, block cache and worker pool are shared by +/// every column family, so their settings live here. A numeric field left at `0` is +/// resolved to the library default for the thread counts, `block_cache_size`, +/// `max_open_sstables`, `vlog_segment_size`, `memtable_write_buffer_size` and the +/// memtable skip-list settings. #[derive(Debug, Clone)] pub struct Config { /// Path to the database directory pub db_path: String, - /// Number of flush threads + /// Number of flush worker threads pub num_flush_threads: i32, - /// Number of compaction threads + /// Number of compaction worker threads pub num_compaction_threads: i32, - /// Logging level + /// Minimum severity to log pub log_level: LogLevel, - /// Block cache size in bytes + /// Size in bytes of the block cache for hot sstable blocks pub block_cache_size: usize, - /// Maximum number of open SSTable files + /// Maximum number of concurrently open sstable file handles. Lowered at open to what + /// the process's open-file ceiling allows; see [`raise_open_file_limit`](crate::raise_open_file_limit). pub max_open_sstables: usize, - /// Global memory limit in bytes (0 = auto, 50% of system RAM; minimum: 5% of system RAM) - pub max_memory_usage: usize, - /// Write logs to file instead of stderr + /// Write the log to a file named `LOG` inside `db_path` instead of stderr. The sink is + /// process-wide: the last database opened with this set owns it. pub log_to_file: bool, - /// Log file truncation threshold in bytes (0 = no truncation) + /// Size in bytes past which the log file is truncated (0 = never) pub log_truncation_at: usize, - /// Enable unified memtable mode (default: false = per-CF memtables) - pub unified_memtable: bool, - /// Unified memtable write buffer size (0 = auto) - pub unified_memtable_write_buffer_size: usize, - /// Skip list max level for unified memtable (0 = default 12) - pub unified_memtable_skip_list_max_level: i32, - /// Skip list probability for unified memtable (0 = default 0.25) - pub unified_memtable_skip_list_probability: f32, - /// Sync mode for unified WAL (default: SyncMode::None) - pub unified_memtable_sync_mode: SyncMode, - /// Sync interval for unified WAL in microseconds (0 = default) - pub unified_memtable_sync_interval_us: u64, - /// Filesystem root directory for the object store connector (None = no object store) - pub object_store_fs_path: Option, - /// S3 object store connector configuration (None = no S3 object store). - /// Takes precedence over `object_store_fs_path` when both are set. - #[cfg(feature = "objectstore")] - pub object_store_s3: Option, - /// Object store behavior configuration (None = use defaults when object store is set) - pub object_store_config: Option, - /// Global semaphore on the number of in-flight memtable flushes across all column - /// families. Bounds peak transient memory and work-queue depth where supported. - /// `0` uses the C library's version-specific default behavior. - pub max_concurrent_flushes: i32, + /// Memory the active memtable may occupy before it is rotated (0 = auto) + pub memtable_write_buffer_size: usize, + /// Memtable skip list max level (0 = default 12) + pub memtable_skip_list_max_level: i32, + /// Memtable skip list level probability (0 = default 0.25) + pub memtable_skip_list_probability: f32, + /// Durability mode for the write-ahead log + pub memtable_sync_mode: SyncMode, + /// fsync interval in microseconds for [`SyncMode::Interval`] (0 = one second) + pub memtable_sync_interval_us: u64, + /// Values at or above this size go to the shared value log (0 = default). Keep it at or + /// under a quarter of a column family's `btree_klog_block_size`. + pub value_separation_threshold: usize, + /// Size at which the value log seals a segment and opens a new one (0 = default) + pub vlog_segment_size: usize, + /// Immutable-memtable queue depth at which writes stall for backpressure (0 = never) + pub memtable_l0_queue_stall_threshold: i32, + /// Seconds the active memtable may sit unwritten before it is rotated (0 = never) + pub memtable_idle_flush_seconds: i32, + /// Seconds a transaction may stay active before its next operation expires it (0 = no timeout) + pub txn_timeout_seconds: i64, } impl Config { @@ -443,19 +201,19 @@ impl Config { } } - /// Set the number of flush threads. + /// Set the number of flush worker threads. pub fn num_flush_threads(mut self, n: i32) -> Self { self.num_flush_threads = n; self } - /// Set the number of compaction threads. + /// Set the number of compaction worker threads. pub fn num_compaction_threads(mut self, n: i32) -> Self { self.num_compaction_threads = n; self } - /// Set the logging level. + /// Set the logging threshold. pub fn log_level(mut self, level: LogLevel) -> Self { self.log_level = level; self @@ -467,334 +225,188 @@ impl Config { self } - /// Set the maximum number of open SSTable files. + /// Set the maximum number of open sstable file handles. pub fn max_open_sstables(mut self, n: usize) -> Self { self.max_open_sstables = n; self } - /// Set the global memory limit in bytes. - /// 0 = auto (50% of system RAM). Minimum: 5% of system RAM. - pub fn max_memory_usage(mut self, size: usize) -> Self { - self.max_memory_usage = size; - self - } - - /// Enable writing logs to a file instead of stderr. + /// Write logs to `/LOG` instead of stderr. pub fn log_to_file(mut self, enable: bool) -> Self { self.log_to_file = enable; self } - /// Set the log file truncation threshold in bytes (0 = no truncation). + /// Set the log file truncation threshold in bytes (0 = never). pub fn log_truncation_at(mut self, size: usize) -> Self { self.log_truncation_at = size; self } - /// Enable or disable unified memtable mode. - /// When enabled, all column families share a single memtable and WAL. - pub fn unified_memtable(mut self, enable: bool) -> Self { - self.unified_memtable = enable; + /// Set the memtable write buffer size in bytes (0 = auto). + pub fn memtable_write_buffer_size(mut self, size: usize) -> Self { + self.memtable_write_buffer_size = size; self } - /// Set the unified memtable write buffer size (0 = auto). - pub fn unified_memtable_write_buffer_size(mut self, size: usize) -> Self { - self.unified_memtable_write_buffer_size = size; + /// Set the memtable skip list max level (0 = default 12). + pub fn memtable_skip_list_max_level(mut self, level: i32) -> Self { + self.memtable_skip_list_max_level = level; self } - /// Set the skip list max level for unified memtable (0 = default 12). - pub fn unified_memtable_skip_list_max_level(mut self, level: i32) -> Self { - self.unified_memtable_skip_list_max_level = level; + /// Set the memtable skip list level probability (0 = default 0.25). + pub fn memtable_skip_list_probability(mut self, prob: f32) -> Self { + self.memtable_skip_list_probability = prob; self } - /// Set the skip list probability for unified memtable (0 = default 0.25). - pub fn unified_memtable_skip_list_probability(mut self, prob: f32) -> Self { - self.unified_memtable_skip_list_probability = prob; + /// Set the write-ahead log durability mode. + pub fn memtable_sync_mode(mut self, mode: SyncMode) -> Self { + self.memtable_sync_mode = mode; self } - /// Set the sync mode for unified WAL. - pub fn unified_memtable_sync_mode(mut self, mode: SyncMode) -> Self { - self.unified_memtable_sync_mode = mode; + /// Set the fsync interval for [`SyncMode::Interval`], in microseconds. + pub fn memtable_sync_interval_us(mut self, interval: u64) -> Self { + self.memtable_sync_interval_us = interval; self } - /// Set the sync interval for unified WAL in microseconds. - pub fn unified_memtable_sync_interval_us(mut self, interval: u64) -> Self { - self.unified_memtable_sync_interval_us = interval; + /// Set the size at or above which values are stored in the value log (0 = default). + pub fn value_separation_threshold(mut self, size: usize) -> Self { + self.value_separation_threshold = size; self } - /// Enable object store mode with a filesystem connector. - /// - /// Stores objects as files under `root_dir` mirroring the key path structure. - /// Useful for testing and local replication (e.g., NFS mount). - /// - /// Object store mode automatically enables unified memtable mode. - pub fn object_store_fs(mut self, root_dir: &str) -> Self { - self.object_store_fs_path = Some(root_dir.to_string()); + /// Set the value log segment size in bytes (0 = default). + pub fn vlog_segment_size(mut self, size: usize) -> Self { + self.vlog_segment_size = size; self } - /// Enable object store mode with an S3-compatible connector (AWS S3, MinIO, etc.). - /// - /// Requires the `objectstore` Cargo feature and a TidesDB C library built with - /// `TIDESDB_WITH_S3=ON`. Takes precedence over [`object_store_fs`](Self::object_store_fs) - /// when both are set. Object store mode automatically enables unified memtable mode. - #[cfg(feature = "objectstore")] - pub fn object_store_s3(mut self, config: S3Config) -> Self { - self.object_store_s3 = Some(config); + /// Set the immutable-queue depth at which writes stall (0 = never stall). + pub fn memtable_l0_queue_stall_threshold(mut self, threshold: i32) -> Self { + self.memtable_l0_queue_stall_threshold = threshold; self } - /// Set the object store behavior configuration. - /// - /// Controls caching, upload/download parallelism, WAL replication, - /// replica mode, and other object store behavior. If not set when an - /// object store connector is configured, defaults are used. - pub fn object_store_config(mut self, config: ObjectStoreConfig) -> Self { - self.object_store_config = Some(config); + /// Set how long the memtable may sit idle before it is rotated (0 = never). + pub fn memtable_idle_flush_seconds(mut self, seconds: i32) -> Self { + self.memtable_idle_flush_seconds = seconds; self } - /// Set the global cap on in-flight memtable flushes across all column families. - /// - /// Bounds peak transient memory and work-queue depth when many column families - /// flush at once. `0` falls back to the library default. - pub fn max_concurrent_flushes(mut self, n: i32) -> Self { - self.max_concurrent_flushes = n; + /// Set the default transaction timeout in seconds (0 = no timeout). + pub fn txn_timeout_seconds(mut self, seconds: i64) -> Self { + self.txn_timeout_seconds = seconds; self } - /// Convert to C configuration struct. - /// Returns a `CConfigData` that owns all heap allocations needed during `tidesdb_open`. - pub(crate) fn to_c_config(&self) -> crate::error::Result { + /// Convert to the C configuration struct. + pub(crate) fn to_c_config(&self) -> Result { let c_path = CString::new(self.db_path.as_str())?; - // Build object store connector if configured - let objstore_ptr = match &self.object_store_fs_path { - Some(root_dir) => { - let c_root = CString::new(root_dir.as_str())?; - let ptr = unsafe { ffi::tidesdb_objstore_fs_create(c_root.as_ptr()) }; - if ptr.is_null() { - return Err(crate::error::Error::NullPointer("object store connector")); - } - ptr - } - None => std::ptr::null_mut(), - }; - - // An S3 connector takes precedence over the filesystem connector. - // The C factory copies all fields, so the temporary CStrings need only - // outlive the call itself. - #[cfg(feature = "objectstore")] - let objstore_ptr = if let Some(s3) = &self.object_store_s3 { - let c_endpoint = CString::new(s3.endpoint.as_str())?; - let c_bucket = CString::new(s3.bucket.as_str())?; - let c_access = CString::new(s3.access_key.as_str())?; - let c_secret = CString::new(s3.secret_key.as_str())?; - let c_prefix = s3.prefix.as_deref().map(CString::new).transpose()?; - let c_region = s3.region.as_deref().map(CString::new).transpose()?; - let c_ca = s3.tls_ca_path.as_deref().map(CString::new).transpose()?; - - let s3_c_config = ffi::tidesdb_objstore_s3_config_t { - endpoint: c_endpoint.as_ptr(), - bucket: c_bucket.as_ptr(), - prefix: c_prefix.as_ref().map_or(std::ptr::null(), |c| c.as_ptr()), - access_key: c_access.as_ptr(), - secret_key: c_secret.as_ptr(), - region: c_region.as_ref().map_or(std::ptr::null(), |c| c.as_ptr()), - use_ssl: if s3.use_ssl { 1 } else { 0 }, - use_path_style: if s3.use_path_style { 1 } else { 0 }, - tls_ca_path: c_ca.as_ref().map_or(std::ptr::null(), |c| c.as_ptr()), - tls_insecure_skip_verify: if s3.tls_insecure_skip_verify { 1 } else { 0 }, - multipart_threshold: s3.multipart_threshold, - multipart_part_size: s3.multipart_part_size, - }; - - let ptr = unsafe { ffi::tidesdb_objstore_s3_create_config(&s3_c_config) }; - if ptr.is_null() { - return Err(crate::error::Error::NullPointer("S3 object store connector")); - } - ptr - } else { - objstore_ptr - }; - - // Build object store config if configured (or if connector is set, use defaults) - let (boxed_os_config, cache_path_cstr) = if !objstore_ptr.is_null() { - let os_cfg = self - .object_store_config - .as_ref() - .cloned() - .unwrap_or_default(); - let (c_os_config, cache_cstr) = os_cfg.to_c_config(); - (Some(Box::new(c_os_config)), cache_cstr) - } else { - (None, None) - }; - - let os_config_ptr = match &boxed_os_config { - Some(b) => &**b as *const ffi::tidesdb_objstore_config_t as *mut _, - None => std::ptr::null_mut(), - }; - let config = ffi::tidesdb_config_t { - db_path: c_path.as_ptr(), + // borrowed by tidesdb_open, which copies it + db_path: c_path.as_ptr() as *mut _, num_flush_threads: self.num_flush_threads, num_compaction_threads: self.num_compaction_threads, log_level: self.log_level as i32, block_cache_size: self.block_cache_size, max_open_sstables: self.max_open_sstables, - log_to_file: if self.log_to_file { 1 } else { 0 }, + log_to_file: self.log_to_file as i32, log_truncation_at: self.log_truncation_at, - max_memory_usage: self.max_memory_usage, - unified_memtable: if self.unified_memtable { 1 } else { 0 }, - unified_memtable_write_buffer_size: self.unified_memtable_write_buffer_size, - unified_memtable_skip_list_max_level: self.unified_memtable_skip_list_max_level, - unified_memtable_skip_list_probability: self.unified_memtable_skip_list_probability, - unified_memtable_sync_mode: self.unified_memtable_sync_mode as i32, - unified_memtable_sync_interval_us: self.unified_memtable_sync_interval_us, - object_store: objstore_ptr, - object_store_config: os_config_ptr, - #[cfg(tidesdb_has_max_concurrent_flushes)] - max_concurrent_flushes: self.max_concurrent_flushes, + memtable_write_buffer_size: self.memtable_write_buffer_size, + memtable_skip_list_max_level: self.memtable_skip_list_max_level, + memtable_skip_list_probability: self.memtable_skip_list_probability, + memtable_sync_mode: self.memtable_sync_mode as i32, + memtable_sync_interval_us: self.memtable_sync_interval_us, + value_separation_threshold: self.value_separation_threshold, + vlog_segment_size: self.vlog_segment_size, + memtable_l0_queue_stall_threshold: self.memtable_l0_queue_stall_threshold, + memtable_idle_flush_seconds: self.memtable_idle_flush_seconds, + txn_timeout_seconds: self.txn_timeout_seconds, }; Ok(CConfigData { config, _db_path: c_path, - _objstore_config: boxed_os_config, - _cache_path: cache_path_cstr, }) } } impl Default for Config { fn default() -> Self { - // Pull defaults from the C library so the binding tracks engine - // defaults automatically. `db_path` is left empty; callers must set it - // before opening (`Config::new` and the `db_path` builder do this). + // Pull defaults from the C library so the binding tracks engine defaults + // automatically. `db_path` is left empty; `Config::new` sets it. let c = unsafe { ffi::tidesdb_default_config() }; Config { db_path: String::new(), num_flush_threads: c.num_flush_threads, num_compaction_threads: c.num_compaction_threads, - log_level: match c.log_level { - ffi::TDB_LOG_DEBUG => LogLevel::Debug, - ffi::TDB_LOG_WARN => LogLevel::Warn, - ffi::TDB_LOG_ERROR => LogLevel::Error, - ffi::TDB_LOG_FATAL => LogLevel::Fatal, - ffi::TDB_LOG_NONE => LogLevel::None, - _ => LogLevel::Info, - }, + log_level: LogLevel::from_c(c.log_level), block_cache_size: c.block_cache_size, max_open_sstables: c.max_open_sstables, - max_memory_usage: c.max_memory_usage, log_to_file: c.log_to_file != 0, log_truncation_at: c.log_truncation_at, - unified_memtable: c.unified_memtable != 0, - unified_memtable_write_buffer_size: c.unified_memtable_write_buffer_size, - unified_memtable_skip_list_max_level: c.unified_memtable_skip_list_max_level, - unified_memtable_skip_list_probability: c.unified_memtable_skip_list_probability, - unified_memtable_sync_mode: match c.unified_memtable_sync_mode { - ffi::TDB_SYNC_FULL => SyncMode::Full, - ffi::TDB_SYNC_INTERVAL => SyncMode::Interval, - _ => SyncMode::None, - }, - unified_memtable_sync_interval_us: c.unified_memtable_sync_interval_us, - object_store_fs_path: None, - #[cfg(feature = "objectstore")] - object_store_s3: None, - object_store_config: None, - max_concurrent_flushes: { - #[cfg(tidesdb_has_max_concurrent_flushes)] - { - c.max_concurrent_flushes - } - #[cfg(not(tidesdb_has_max_concurrent_flushes))] - { - 0 - } - }, + memtable_write_buffer_size: c.memtable_write_buffer_size, + memtable_skip_list_max_level: c.memtable_skip_list_max_level, + memtable_skip_list_probability: c.memtable_skip_list_probability, + memtable_sync_mode: SyncMode::from_c(c.memtable_sync_mode), + memtable_sync_interval_us: c.memtable_sync_interval_us, + value_separation_threshold: c.value_separation_threshold, + vlog_segment_size: c.vlog_segment_size, + memtable_l0_queue_stall_threshold: c.memtable_l0_queue_stall_threshold, + memtable_idle_flush_seconds: c.memtable_idle_flush_seconds, + txn_timeout_seconds: c.txn_timeout_seconds, } } } /// Column family configuration. -#[derive(Debug, Clone)] +/// +/// Every field can be changed at runtime with +/// [`ColumnFamily::update_runtime_config`](crate::ColumnFamily::update_runtime_config), +/// since keys are always ordered byte-wise and sstables stay mergeable. +#[derive(Debug, Clone, PartialEq)] pub struct ColumnFamilyConfig { - /// Write buffer size in bytes - pub write_buffer_size: usize, - /// Level size ratio + /// Target size ratio between successive levels pub level_size_ratio: usize, - /// Minimum number of levels + /// Floor on the level count pub min_levels: i32, - /// Dividing level offset + /// How far above the largest level the dividing level sits pub dividing_level_offset: i32, - /// Key-log value threshold - pub klog_value_threshold: usize, - /// Compression algorithm - pub compression_algorithm: CompressionAlgorithm, - /// Enable bloom filter + /// Keep every value in the key log regardless of the database's + /// `value_separation_threshold` + pub keep_values_inline: bool, + /// Target btree key-log node size in bytes (0 = let the btree choose its own, larger, default) + pub btree_klog_block_size: usize, + /// Encodings applied in order to key-log nodes and separated values, at most + /// 8 stages. Empty means data is stored verbatim. + pub encoding_pipeline: Vec, + /// Build bloom filters for point-get pruning pub enable_bloom_filter: bool, - /// Bloom filter false positive rate + /// Target bloom filter false-positive rate pub bloom_fpr: f64, - /// Enable block indexes - pub enable_block_indexes: bool, - /// Index sample ratio - pub index_sample_ratio: i32, - /// Block index prefix length - pub block_index_prefix_len: i32, - /// Sync mode - pub sync_mode: SyncMode, - /// Sync interval in microseconds - pub sync_interval_us: u64, - /// Comparator name - pub comparator_name: String, - /// Skip list maximum level - pub skip_list_max_level: i32, - /// Skip list probability - pub skip_list_probability: f32, - /// Default isolation level + /// Isolation level for transactions begun with + /// [`TidesDB::begin_transaction_cf`](crate::TidesDB::begin_transaction_cf) pub default_isolation_level: IsolationLevel, - /// Minimum disk space - pub min_disk_space: u64, - /// L1 file count trigger + /// L1 sstable count that triggers compaction pub l1_file_count_trigger: i32, - /// L0 queue stall threshold - pub l0_queue_stall_threshold: i32, - /// Per-SSTable tombstone density (`tombstone_count / num_entries`) above which compaction - /// priority escalates. Range `[0.0, 1.0]`; `0.0` disables the check. + /// Tombstone density in `[0, 1]` above which an sstable escalates compaction (0 = disabled) pub tombstone_density_trigger: f64, - /// SSTables with fewer entries than this are ignored by the density trigger - /// (prevents tiny-sstable noise). `0` falls back to the library default. + /// Minimum entries for an sstable to be judged by the density trigger (0 = no minimum) pub tombstone_density_min_entries: u64, - /// Use B+tree format for klog (default: false = block-based) - pub use_btree: bool, - /// Compact less aggressively in object store mode (default: false) - pub object_lazy_compaction: bool, - /// Download all inputs before merge in object store mode (default: true) - pub object_prefetch_compaction: bool, } impl ColumnFamilyConfig { - /// Create a new column family configuration with default values. + /// Create a new column family configuration with library defaults. pub fn new() -> Self { Self::default() } - /// Set the write buffer size. - pub fn write_buffer_size(mut self, size: usize) -> Self { - self.write_buffer_size = size; - self - } - - /// Set the level size ratio. + /// Set the size ratio between levels. pub fn level_size_ratio(mut self, ratio: usize) -> Self { self.level_size_ratio = ratio; self @@ -806,405 +418,137 @@ impl ColumnFamilyConfig { self } - /// Set the compression algorithm. - pub fn compression_algorithm(mut self, algo: CompressionAlgorithm) -> Self { - self.compression_algorithm = algo; - self - } - - /// Enable or disable bloom filter. - pub fn enable_bloom_filter(mut self, enable: bool) -> Self { - self.enable_bloom_filter = enable; - self - } - - /// Set the bloom filter false positive rate. - pub fn bloom_fpr(mut self, fpr: f64) -> Self { - self.bloom_fpr = fpr; - self - } - - /// Enable or disable block indexes. - pub fn enable_block_indexes(mut self, enable: bool) -> Self { - self.enable_block_indexes = enable; - self - } - - /// Set the sync mode. - pub fn sync_mode(mut self, mode: SyncMode) -> Self { - self.sync_mode = mode; - self - } - - /// Set the sync interval in microseconds. - pub fn sync_interval_us(mut self, interval: u64) -> Self { - self.sync_interval_us = interval; - self - } - - /// Set the default isolation level. - pub fn default_isolation_level(mut self, level: IsolationLevel) -> Self { - self.default_isolation_level = level; - self - } - /// Set the dividing level offset. pub fn dividing_level_offset(mut self, offset: i32) -> Self { self.dividing_level_offset = offset; self } - /// Set the klog value threshold. - pub fn klog_value_threshold(mut self, threshold: usize) -> Self { - self.klog_value_threshold = threshold; + /// Keep every value inline in the key log. + pub fn keep_values_inline(mut self, enable: bool) -> Self { + self.keep_values_inline = enable; self } - /// Set the index sample ratio. - pub fn index_sample_ratio(mut self, ratio: i32) -> Self { - self.index_sample_ratio = ratio; + /// Set the btree key-log node size in bytes. + pub fn btree_klog_block_size(mut self, size: usize) -> Self { + self.btree_klog_block_size = size; self } - /// Set the block index prefix length. - pub fn block_index_prefix_len(mut self, len: i32) -> Self { - self.block_index_prefix_len = len; + /// Compress with a single algorithm; [`CompressionAlgorithm::None`] clears the pipeline. + pub fn compression_algorithm(mut self, algo: CompressionAlgorithm) -> Self { + self.encoding_pipeline = match algo { + CompressionAlgorithm::None => Vec::new(), + other => vec![other], + }; self } - /// Set the comparator name. - pub fn comparator_name(mut self, name: &str) -> Self { - self.comparator_name = name.to_string(); + /// Set the full encoding pipeline, applied in order (at most 8 stages). + pub fn encoding_pipeline(mut self, pipeline: Vec) -> Self { + self.encoding_pipeline = pipeline; self } - /// Set the skip list max level. - pub fn skip_list_max_level(mut self, level: i32) -> Self { - self.skip_list_max_level = level; + /// Enable or disable bloom filters. + pub fn enable_bloom_filter(mut self, enable: bool) -> Self { + self.enable_bloom_filter = enable; self } - /// Set the skip list probability. - pub fn skip_list_probability(mut self, prob: f32) -> Self { - self.skip_list_probability = prob; + /// Set the bloom filter false-positive rate. + pub fn bloom_fpr(mut self, fpr: f64) -> Self { + self.bloom_fpr = fpr; self } - /// Set the minimum disk space required. - pub fn min_disk_space(mut self, space: u64) -> Self { - self.min_disk_space = space; + /// Set the default isolation level. + pub fn default_isolation_level(mut self, level: IsolationLevel) -> Self { + self.default_isolation_level = level; self } - /// Set the L1 file count trigger for compaction. + /// Set the L1 file count that triggers compaction. pub fn l1_file_count_trigger(mut self, trigger: i32) -> Self { self.l1_file_count_trigger = trigger; self } - /// Set the L0 queue stall threshold for backpressure. - pub fn l0_queue_stall_threshold(mut self, threshold: i32) -> Self { - self.l0_queue_stall_threshold = threshold; - self - } - - /// Set the per-SSTable tombstone density above which compaction priority escalates. - /// Range `[0.0, 1.0]`; `0.0` disables the check. + /// Set the tombstone density compaction trigger (0 disables). pub fn tombstone_density_trigger(mut self, ratio: f64) -> Self { self.tombstone_density_trigger = ratio; self } - /// Set the minimum entry count for an SSTable to be considered by the tombstone - /// density trigger. SSTables with fewer entries are skipped. + /// Set the minimum entry count for the tombstone density trigger. pub fn tombstone_density_min_entries(mut self, n: u64) -> Self { self.tombstone_density_min_entries = n; self } - /// Enable or disable B+tree format for klog. - /// When enabled, uses B+tree structure instead of block-based format. - pub fn use_btree(mut self, enable: bool) -> Self { - self.use_btree = enable; - self - } - - /// Enable or disable lazy compaction in object store mode. - pub fn object_lazy_compaction(mut self, enable: bool) -> Self { - self.object_lazy_compaction = enable; - self - } - - /// Enable or disable prefetch compaction in object store mode. - pub fn object_prefetch_compaction(mut self, enable: bool) -> Self { - self.object_prefetch_compaction = enable; - self - } - - /// Load configuration from an INI file. - /// - /// # Arguments - /// - /// * `ini_file` - Path to the INI file - /// * `section_name` - Section name in the INI file - pub fn load_from_ini(ini_file: &str, section_name: &str) -> Result { - let c_ini_file = CString::new(ini_file)?; - let c_section_name = CString::new(section_name)?; - let mut c_config = unsafe { ffi::tidesdb_default_column_family_config() }; - - let result = unsafe { - ffi::tidesdb_cf_config_load_from_ini( - c_ini_file.as_ptr(), - c_section_name.as_ptr(), - &mut c_config, - ) - }; - check_result(result, "failed to load config from INI")?; - - Ok(Self::from_c_config(&c_config)) - } - - /// Save configuration to an INI file. - /// - /// # Arguments - /// - /// * `ini_file` - Path to the INI file - /// * `section_name` - Section name in the INI file - pub fn save_to_ini(&self, ini_file: &str, section_name: &str) -> Result<()> { - let c_ini_file = CString::new(ini_file)?; - let c_section_name = CString::new(section_name)?; - let c_config = self.to_c_config(); - - let result = unsafe { - ffi::tidesdb_cf_config_save_to_ini( - c_ini_file.as_ptr(), - c_section_name.as_ptr(), - &c_config, - ) - }; - check_result(result, "failed to save config to INI") - } - - /// Create a ColumnFamilyConfig by reading from a C config struct pointer. + /// Convert to the C column family configuration struct. /// - /// # Safety - /// - /// `ptr` must point to a valid `tidesdb_column_family_config_t`. This is - /// crate-internal; callers (e.g., the stats readback) are expected to - /// validate the pointer before invoking. - pub(crate) fn from_c_config_ptr(ptr: *const ffi::tidesdb_column_family_config_t) -> Self { - let c_config = unsafe { &*ptr }; - Self::from_c_config(c_config) - } - - /// Create a ColumnFamilyConfig from a C config struct. - fn from_c_config(c_config: &ffi::tidesdb_column_family_config_t) -> Self { - let mut comparator_name = String::new(); - let name_bytes: Vec = c_config - .comparator_name + /// The name and commit hook are left empty: the name is passed separately and the + /// hook is set with [`ColumnFamily::set_commit_hook`](crate::ColumnFamily::set_commit_hook). + pub(crate) fn to_c_config(&self) -> Result { + let stages: Vec = self + .encoding_pipeline .iter() - .take_while(|&&c| c != 0) - .map(|&c| c as u8) + .filter(|a| **a != CompressionAlgorithm::None) + .map(|a| *a as i32 as u8) .collect(); - if let Ok(s) = std::str::from_utf8(&name_bytes) { - comparator_name = s.to_string(); + if stages.len() > ffi::TDB_ENCODING_PIPELINE_MAX { + return Err(Error::from_code( + ffi::TDB_ERR_INVALID_ARGS, + "encoding pipeline has more than 8 stages", + )); } + let mut c = unsafe { ffi::tidesdb_default_column_family_config() }; + c.level_size_ratio = self.level_size_ratio; + c.min_levels = self.min_levels; + c.dividing_level_offset = self.dividing_level_offset; + c.keep_values_inline = self.keep_values_inline as i32; + c.btree_klog_block_size = self.btree_klog_block_size; + c.encoding_pipeline = [0; ffi::TDB_ENCODING_PIPELINE_MAX]; + c.encoding_pipeline[..stages.len()].copy_from_slice(&stages); + c.encoding_count = stages.len() as u8; + c.enable_bloom_filter = self.enable_bloom_filter as i32; + c.bloom_fpr = self.bloom_fpr; + c.default_isolation_level = self.default_isolation_level as i32; + c.l1_file_count_trigger = self.l1_file_count_trigger; + c.tombstone_density_trigger = self.tombstone_density_trigger; + c.tombstone_density_min_entries = self.tombstone_density_min_entries; + Ok(c) + } + + /// Build from a C column family configuration. + pub(crate) fn from_c_config(c: &ffi::tidesdb_column_family_config_t) -> Self { + let count = (c.encoding_count as usize).min(ffi::TDB_ENCODING_PIPELINE_MAX); ColumnFamilyConfig { - write_buffer_size: c_config.write_buffer_size, - level_size_ratio: c_config.level_size_ratio, - min_levels: c_config.min_levels, - dividing_level_offset: c_config.dividing_level_offset, - klog_value_threshold: c_config.klog_value_threshold, - compression_algorithm: match c_config.compression_algo { - ffi::SNAPPY_COMPRESSION => CompressionAlgorithm::Snappy, - ffi::LZ4_COMPRESSION => CompressionAlgorithm::Lz4, - ffi::ZSTD_COMPRESSION => CompressionAlgorithm::Zstd, - ffi::LZ4_FAST_COMPRESSION => CompressionAlgorithm::Lz4Fast, - _ => CompressionAlgorithm::None, - }, - enable_bloom_filter: c_config.enable_bloom_filter != 0, - bloom_fpr: c_config.bloom_fpr, - enable_block_indexes: c_config.enable_block_indexes != 0, - index_sample_ratio: c_config.index_sample_ratio, - block_index_prefix_len: c_config.block_index_prefix_len, - sync_mode: match c_config.sync_mode { - ffi::TDB_SYNC_FULL => SyncMode::Full, - ffi::TDB_SYNC_INTERVAL => SyncMode::Interval, - _ => SyncMode::None, - }, - sync_interval_us: c_config.sync_interval_us, - comparator_name, - skip_list_max_level: c_config.skip_list_max_level, - skip_list_probability: c_config.skip_list_probability, - default_isolation_level: match c_config.default_isolation_level { - ffi::TDB_ISOLATION_READ_UNCOMMITTED => IsolationLevel::ReadUncommitted, - ffi::TDB_ISOLATION_REPEATABLE_READ => IsolationLevel::RepeatableRead, - ffi::TDB_ISOLATION_SNAPSHOT => IsolationLevel::Snapshot, - ffi::TDB_ISOLATION_SERIALIZABLE => IsolationLevel::Serializable, - _ => IsolationLevel::ReadCommitted, - }, - min_disk_space: c_config.min_disk_space, - l1_file_count_trigger: c_config.l1_file_count_trigger, - l0_queue_stall_threshold: c_config.l0_queue_stall_threshold, - tombstone_density_trigger: { - #[cfg(tidesdb_has_tombstone_stats)] - { - c_config.tombstone_density_trigger - } - #[cfg(not(tidesdb_has_tombstone_stats))] - { - 0.0 - } - }, - tombstone_density_min_entries: { - #[cfg(tidesdb_has_tombstone_stats)] - { - c_config.tombstone_density_min_entries - } - #[cfg(not(tidesdb_has_tombstone_stats))] - { - 0 - } - }, - use_btree: c_config.use_btree != 0, - object_lazy_compaction: c_config.object_lazy_compaction != 0, - object_prefetch_compaction: c_config.object_prefetch_compaction != 0, + level_size_ratio: c.level_size_ratio, + min_levels: c.min_levels, + dividing_level_offset: c.dividing_level_offset, + keep_values_inline: c.keep_values_inline != 0, + btree_klog_block_size: c.btree_klog_block_size, + encoding_pipeline: c.encoding_pipeline[..count] + .iter() + .filter_map(|id| CompressionAlgorithm::from_id(*id)) + .collect(), + enable_bloom_filter: c.enable_bloom_filter != 0, + bloom_fpr: c.bloom_fpr, + default_isolation_level: IsolationLevel::from_c(c.default_isolation_level), + l1_file_count_trigger: c.l1_file_count_trigger, + tombstone_density_trigger: c.tombstone_density_trigger, + tombstone_density_min_entries: c.tombstone_density_min_entries, } } - - /// Convert to C configuration struct. - pub(crate) fn to_c_config(&self) -> ffi::tidesdb_column_family_config_t { - let mut config = ffi::tidesdb_column_family_config_t { - name: [0; ffi::TDB_MAX_CF_NAME_LEN], - write_buffer_size: self.write_buffer_size, - level_size_ratio: self.level_size_ratio, - min_levels: self.min_levels, - dividing_level_offset: self.dividing_level_offset, - klog_value_threshold: self.klog_value_threshold, - compression_algo: self.compression_algorithm as i32, - enable_bloom_filter: if self.enable_bloom_filter { 1 } else { 0 }, - bloom_fpr: self.bloom_fpr, - enable_block_indexes: if self.enable_block_indexes { 1 } else { 0 }, - index_sample_ratio: self.index_sample_ratio, - block_index_prefix_len: self.block_index_prefix_len, - sync_mode: self.sync_mode as i32, - sync_interval_us: self.sync_interval_us, - comparator_name: [0; ffi::TDB_MAX_COMPARATOR_NAME], - comparator_ctx_str: [0; ffi::TDB_MAX_COMPARATOR_CTX], - comparator_fn_cached: std::ptr::null_mut(), - comparator_ctx_cached: std::ptr::null_mut(), - skip_list_max_level: self.skip_list_max_level, - skip_list_probability: self.skip_list_probability, - default_isolation_level: self.default_isolation_level as i32, - min_disk_space: self.min_disk_space, - l1_file_count_trigger: self.l1_file_count_trigger, - l0_queue_stall_threshold: self.l0_queue_stall_threshold, - #[cfg(tidesdb_has_tombstone_stats)] - tombstone_density_trigger: self.tombstone_density_trigger, - #[cfg(tidesdb_has_tombstone_stats)] - tombstone_density_min_entries: self.tombstone_density_min_entries, - use_btree: if self.use_btree { 1 } else { 0 }, - commit_hook_fn: None, - commit_hook_ctx: std::ptr::null_mut(), - object_target_file_size: 0, - object_lazy_compaction: if self.object_lazy_compaction { 1 } else { 0 }, - object_prefetch_compaction: if self.object_prefetch_compaction { - 1 - } else { - 0 - }, - }; - - // Copy comparator name - if !self.comparator_name.is_empty() { - let bytes = self.comparator_name.as_bytes(); - let len = bytes.len().min(ffi::TDB_MAX_COMPARATOR_NAME - 1); - for (i, &b) in bytes[..len].iter().enumerate() { - config.comparator_name[i] = b as libc::c_char; - } - } - - config - } } impl Default for ColumnFamilyConfig { fn default() -> Self { - // Get defaults from C library - let c_config = unsafe { ffi::tidesdb_default_column_family_config() }; - - let mut comparator_name = String::new(); - let name_bytes: Vec = c_config - .comparator_name - .iter() - .take_while(|&&c| c != 0) - .map(|&c| c as u8) - .collect(); - if let Ok(s) = std::str::from_utf8(&name_bytes) { - comparator_name = s.to_string(); - } - - ColumnFamilyConfig { - write_buffer_size: c_config.write_buffer_size, - level_size_ratio: c_config.level_size_ratio, - min_levels: c_config.min_levels, - dividing_level_offset: c_config.dividing_level_offset, - klog_value_threshold: c_config.klog_value_threshold, - compression_algorithm: match c_config.compression_algo { - ffi::SNAPPY_COMPRESSION => CompressionAlgorithm::Snappy, - ffi::LZ4_COMPRESSION => CompressionAlgorithm::Lz4, - ffi::ZSTD_COMPRESSION => CompressionAlgorithm::Zstd, - ffi::LZ4_FAST_COMPRESSION => CompressionAlgorithm::Lz4Fast, - _ => CompressionAlgorithm::None, - }, - enable_bloom_filter: c_config.enable_bloom_filter != 0, - bloom_fpr: c_config.bloom_fpr, - enable_block_indexes: c_config.enable_block_indexes != 0, - index_sample_ratio: c_config.index_sample_ratio, - block_index_prefix_len: c_config.block_index_prefix_len, - sync_mode: match c_config.sync_mode { - ffi::TDB_SYNC_FULL => SyncMode::Full, - ffi::TDB_SYNC_INTERVAL => SyncMode::Interval, - _ => SyncMode::None, - }, - sync_interval_us: c_config.sync_interval_us, - comparator_name, - skip_list_max_level: c_config.skip_list_max_level, - skip_list_probability: c_config.skip_list_probability, - default_isolation_level: match c_config.default_isolation_level { - ffi::TDB_ISOLATION_READ_UNCOMMITTED => IsolationLevel::ReadUncommitted, - ffi::TDB_ISOLATION_REPEATABLE_READ => IsolationLevel::RepeatableRead, - ffi::TDB_ISOLATION_SNAPSHOT => IsolationLevel::Snapshot, - ffi::TDB_ISOLATION_SERIALIZABLE => IsolationLevel::Serializable, - _ => IsolationLevel::ReadCommitted, - }, - min_disk_space: c_config.min_disk_space, - l1_file_count_trigger: c_config.l1_file_count_trigger, - l0_queue_stall_threshold: c_config.l0_queue_stall_threshold, - tombstone_density_trigger: { - #[cfg(tidesdb_has_tombstone_stats)] - { - c_config.tombstone_density_trigger - } - #[cfg(not(tidesdb_has_tombstone_stats))] - { - 0.0 - } - }, - tombstone_density_min_entries: { - #[cfg(tidesdb_has_tombstone_stats)] - { - c_config.tombstone_density_min_entries - } - #[cfg(not(tidesdb_has_tombstone_stats))] - { - 0 - } - }, - use_btree: c_config.use_btree != 0, - object_lazy_compaction: c_config.object_lazy_compaction != 0, - object_prefetch_compaction: c_config.object_prefetch_compaction != 0, - } + let c = unsafe { ffi::tidesdb_default_column_family_config() }; + Self::from_c_config(&c) } } diff --git a/src/db.rs b/src/db.rs index 681a311..dabf5a9 100644 --- a/src/db.rs +++ b/src/db.rs @@ -8,216 +8,170 @@ use crate::config::{ColumnFamilyConfig, Config, IsolationLevel}; use crate::error::{Error, Result, check_result}; use crate::ffi; -use crate::stats::{CacheStats, DbStats}; +use crate::stats::{CacheStats, CfStats, DbStats, EncodingStats, IoStats, RangeStats, StallStats}; use crate::transaction::Transaction; -use libc::{c_char, c_int, c_void, size_t}; +use libc::{c_char, c_int, c_void}; use std::ffi::{CStr, CString}; +use std::mem::MaybeUninit; +use std::path::Path; use std::ptr; +use std::sync::Mutex; /// A single operation from a committed transaction batch. /// /// Passed to commit hook callbacks. Contains owned copies of the key and value data. -#[derive(Debug, Clone)] +/// +/// | Operation | `key` | `value` | `range_end` | `is_delete` | +/// |-----------|-------|---------|-------------|-------------| +/// | `put` / `put_with_ttl` | the key | `Some(value)` | `None` | `false` | +/// | `delete` / `single_delete` | the key | `None` | `None` | `true` | +/// | `delete_range(lo, Some(hi))` | `lo` | `None` | `Some(hi)` | `true` | +/// | `delete_prefix(p)` | `p` | `None` | `Some(first key past p)` | `true` | +/// | `delete_range(lo, None)` | `lo` | `None` | `None` | `true` | +/// +/// A range delete with no upper bound (and a prefix delete of a prefix made only of `0xFF` +/// bytes, which has no key past it) arrives exactly like a point delete of its lower bound: +/// the C library passes nothing that tells them apart. +#[derive(Debug, Clone, PartialEq, Eq)] pub struct CommitOp { - /// The key + /// The key, or the inclusive lower bound of a range delete pub key: Vec, /// The value (`None` for delete operations) pub value: Option>, - /// TTL (time-to-live) as Unix timestamp, 0 means no expiry - pub ttl: i64, - /// Whether this is a delete operation + /// The exclusive upper bound of a range or prefix delete: every key in + /// `[key, range_end)` was deleted. `None` for every other operation. + pub range_end: Option>, + /// Absolute expiry as a Unix timestamp in seconds, `None` if the entry never expires. + /// This is the deadline the engine stored, not the lifetime passed to `put_with_ttl`. + pub expires_at: Option, + /// Whether this is a delete operation (point, range or prefix) pub is_delete: bool, } -/// Type alias for the boxed commit hook callback. -type CommitHookCallback = Box i32 + Send>; - -/// Type alias for the boxed comparator callback. -type ComparatorCallback = Box i32 + Send + Sync>; +/// The boxed commit hook callback. Commits on different threads may invoke the same +/// hook concurrently, hence `Sync`. +type CommitHookCallback = Box i32 + Send + Sync>; -/// Trampoline function that bridges the C comparator callback to the Rust closure. -unsafe extern "C" fn comparator_trampoline( - key1: *const u8, - key1_size: size_t, - key2: *const u8, - key2_size: size_t, +/// Trampoline that bridges the C commit hook to the Rust closure. +unsafe extern "C" fn commit_hook_trampoline( + ops: *const ffi::tidesdb_commit_op_t, + num_ops: c_int, + commit_seq: u64, ctx: *mut c_void, ) -> c_int { if ctx.is_null() { - return 0; + return -1; } + let callback = unsafe { &*(ctx as *const CommitHookCallback) }; - let callback = unsafe { &*(ctx as *const ComparatorCallback) }; - - let k1 = if key1.is_null() || key1_size == 0 { - &[] + let count = if ops.is_null() { + 0 } else { - unsafe { std::slice::from_raw_parts(key1, key1_size) } - }; - let k2 = if key2.is_null() || key2_size == 0 { - &[] - } else { - unsafe { std::slice::from_raw_parts(key2, key2_size) } + num_ops.max(0) as usize }; + let mut rust_ops = Vec::with_capacity(count); + for i in 0..count { + let op = unsafe { &*ops.add(i) }; + let key = unsafe { bytes_from_raw(op.key, op.key_size) }; + let is_delete = op.is_delete != 0; + let bytes = unsafe { bytes_from_raw(op.value, op.value_size) }; + // a range delete carries its upper bound in the value; a point delete has none + let (value, range_end) = match (is_delete, bytes.is_empty()) { + (false, _) => (Some(bytes), None), + (true, true) => (None, None), + (true, false) => (None, Some(bytes)), + }; + rust_ops.push(CommitOp { + key, + value, + range_end, + expires_at: ttl_from_c(op.ttl), + is_delete, + }); + } - callback(k1, k2) + // A panic must not unwind into C; report it as a hook failure instead. + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + callback(&rust_ops, commit_seq) + })) + .unwrap_or(-1) } -/// Initializes TidesDB with the system allocator. -/// -/// This must be called exactly once before any other TidesDB function -/// when using the explicit initialization path. If not called, TidesDB -/// will auto-initialize with the system allocator on the first `TidesDB::open()`. -/// -/// # Returns -/// -/// `Ok(())` on success, or an error if already initialized. -pub fn init() -> Result<()> { - let result = unsafe { ffi::tidesdb_init(None, None, None, None) }; - check_result(result, "failed to initialize TidesDB") -} - -/// Initializes TidesDB with custom C-level memory allocator functions. -/// -/// This is an advanced function for integrating with custom memory managers -/// (e.g., jemalloc, mimalloc, Redis module allocator). Must be called exactly -/// once before any other TidesDB function. -/// -/// # Safety -/// -/// The caller must ensure that the provided function pointers are valid C allocator -/// functions with correct signatures and semantics (malloc/calloc/realloc/free). -/// -/// # Arguments -/// -/// * `malloc_fn` - Custom malloc function -/// * `calloc_fn` - Custom calloc function -/// * `realloc_fn` - Custom realloc function -/// * `free_fn` - Custom free function -pub unsafe fn init_with_allocator( - malloc_fn: ffi::tidesdb_malloc_fn, - calloc_fn: ffi::tidesdb_calloc_fn, - realloc_fn: ffi::tidesdb_realloc_fn, - free_fn: ffi::tidesdb_free_fn, -) -> Result<()> { - let result = unsafe { ffi::tidesdb_init(malloc_fn, calloc_fn, realloc_fn, free_fn) }; - check_result(result, "failed to initialize TidesDB with custom allocator") +/// Copy `len` bytes from a C pointer, treating NULL or zero length as empty. +pub(crate) unsafe fn bytes_from_raw(ptr: *const u8, len: usize) -> Vec { + if ptr.is_null() || len == 0 { + Vec::new() + } else { + unsafe { std::slice::from_raw_parts(ptr, len) }.to_vec() + } } -/// Finalizes TidesDB and resets the allocator. -/// -/// Should be called after all TidesDB operations are complete (all databases closed). -/// After calling this, `init()` or `init_with_allocator()` can be called again. -pub fn finalize() { - unsafe { - ffi::tidesdb_finalize(); +/// Copy a buffer the library allocated for the caller, then free it. +pub(crate) unsafe fn take_c_buffer(ptr: *mut u8, len: usize) -> Vec { + let out = unsafe { bytes_from_raw(ptr, len) }; + if !ptr.is_null() { + unsafe { ffi::tidesdb_free(ptr as *mut c_void) }; } + out } -/// Raises or reports the process open-file ceiling. -/// -/// Call this before opening a database when a larger `max_open_sstables` budget -/// is needed. Passing `0` or a negative value reports the current ceiling -/// without requesting a change. -#[cfg(tidesdb_has_raise_open_file_limit)] -pub fn raise_open_file_limit(desired: libc::c_long) -> libc::c_long { - unsafe { ffi::tidesdb_raise_open_file_limit(desired) } +/// Convert a C absolute expiry (`-1` = never) to an `Option`. +// time_t is not i64 on every target, so the conversion is not always a no-op +#[allow(clippy::useless_conversion)] +pub(crate) fn ttl_from_c(ttl: libc::time_t) -> Option { + if ttl < 0 { None } else { Some(i64::from(ttl)) } } -/// Frees memory allocated by TidesDB. -/// -/// This is primarily useful for FFI scenarios where memory allocated by TidesDB -/// needs to be freed using the same allocator. For normal Rust usage, the safe -/// wrappers handle memory management automatically. +/// Raises the process open-file ceiling toward `desired` descriptors. /// -/// # Safety +/// TidesDB reads the ceiling at open and lowers `max_open_sstables` to fit it, so call +/// this before [`TidesDB::open`] when a larger budget is wanted. Passing `0` or a negative +/// value only reports the current ceiling. A failed or partial raise is not an error. /// -/// The pointer must have been allocated by TidesDB. -pub unsafe fn free(ptr: *mut c_void) { - unsafe { - ffi::tidesdb_free(ptr); - } -} - -/// Trampoline function that bridges the C callback to the Rust closure. -unsafe extern "C" fn commit_hook_trampoline( - ops: *const ffi::tidesdb_commit_op_t, - num_ops: c_int, - commit_seq: u64, - ctx: *mut c_void, -) -> c_int { - if ctx.is_null() || ops.is_null() || num_ops <= 0 { - return -1; - } - - let callback = unsafe { &*(ctx as *const CommitHookCallback) }; - - let mut rust_ops = Vec::with_capacity(num_ops as usize); - for i in 0..num_ops as isize { - let op = unsafe { &*ops.offset(i) }; - let key = unsafe { std::slice::from_raw_parts(op.key, op.key_size) }.to_vec(); - let value = if op.is_delete != 0 || op.value.is_null() { - None - } else { - Some(unsafe { std::slice::from_raw_parts(op.value, op.value_size) }.to_vec()) - }; - rust_ops.push(CommitOp { - key, - value, - ttl: op.ttl as i64, - is_delete: op.is_delete != 0, - }); - } - - callback(&rust_ops, commit_seq) +/// Returns the open-file ceiling in effect after the attempt. +pub fn raise_open_file_limit(desired: i64) -> i64 { + unsafe { ffi::tidesdb_raise_open_file_limit(desired as libc::c_long) as i64 } } /// A TidesDB database instance. /// -/// This is the main entry point for interacting with TidesDB. -/// The database is automatically closed when dropped. +/// The database is closed when dropped. Every [`ColumnFamily`], [`Transaction`], +/// [`Snapshot`] and [`Iterator`](crate::Iterator) borrows the database, so the borrow +/// checker guarantees they are all gone before it closes. /// /// # Example /// /// ```no_run /// use tidesdb::{TidesDB, Config, ColumnFamilyConfig}; /// -/// let config = Config::new("./mydb") -/// .num_flush_threads(2) -/// .num_compaction_threads(2); -/// -/// let db = TidesDB::open(config)?; -/// -/// // Create a column family +/// let db = TidesDB::open(Config::new("./mydb"))?; /// db.create_column_family("my_cf", ColumnFamilyConfig::default())?; -/// -/// // Get the column family /// let cf = db.get_column_family("my_cf")?; /// -/// // Begin a transaction /// let mut txn = db.begin_transaction()?; -/// txn.put(&cf, b"key", b"value", -1)?; +/// txn.put(&cf, b"key", b"value")?; /// txn.commit()?; /// # Ok::<(), tidesdb::Error>(()) /// ``` pub struct TidesDB { - db: *mut ffi::tidesdb_t, + pub(crate) db: *mut ffi::tidesdb_t, + /// Every commit hook ever installed. The C library swaps a hook's function and context + /// as two separate atomic stores, so a committer may still be running a hook just after + /// it was replaced; each callback is therefore kept alive until the database closes. + // boxed so each callback keeps its address when the Vec grows; C holds that address + #[allow(clippy::vec_box)] + hooks: Mutex>>, } -// TidesDB uses internal locking for thread safety +// The C library synchronises every database-level call internally. unsafe impl Send for TidesDB {} unsafe impl Sync for TidesDB {} impl TidesDB { - /// Opens a TidesDB instance with the given configuration. - /// - /// # Arguments + /// Opens or creates a database with the given configuration. /// - /// * `config` - The database configuration - /// - /// # Returns - /// - /// A new TidesDB instance or an error if the database cannot be opened. + /// Returns [`ErrorCode::Locked`](crate::ErrorCode::Locked) if another handle, in this + /// process or another, already has the directory open. pub fn open(config: Config) -> Result { let c_data = config.to_c_config()?; let mut db: *mut ffi::tidesdb_t = ptr::null_mut(); @@ -229,58 +183,49 @@ impl TidesDB { return Err(Error::NullPointer("database handle")); } - Ok(TidesDB { db }) + Ok(TidesDB { + db, + hooks: Mutex::new(Vec::new()), + }) } + // ===== column families ===== + /// Creates a new column family with the given configuration. - /// - /// # Arguments - /// - /// * `name` - The name of the column family - /// * `config` - The column family configuration pub fn create_column_family(&self, name: &str, config: ColumnFamilyConfig) -> Result<()> { let c_name = CString::new(name)?; - let c_config = config.to_c_config(); + let c_config = config.to_c_config()?; let result = unsafe { ffi::tidesdb_create_column_family(self.db, c_name.as_ptr(), &c_config) }; check_result(result, "failed to create column family") } - /// Drops a column family and all associated data. - /// - /// # Arguments + /// Drops a column family and deletes all of its data. This cannot be undone. /// - /// * `name` - The name of the column family to drop - pub fn drop_column_family(&self, name: &str) -> Result<()> { - let c_name = CString::new(name)?; - - let result = unsafe { ffi::tidesdb_drop_column_family(self.db, c_name.as_ptr()) }; - check_result(result, "failed to drop column family") + /// Takes `&mut self` because dropping invalidates every handle to the family; the + /// borrow checker proves none are alive. To drop a family on a database shared across + /// threads, see [`drop_column_family_unchecked`](Self::drop_column_family_unchecked). + pub fn drop_column_family(&mut self, name: &str) -> Result<()> { + unsafe { self.drop_column_family_unchecked(name) } } - /// Deletes a column family by pointer, skipping the name lookup. - /// - /// This is faster than `drop_column_family` when you already hold a `ColumnFamily`. - /// The `ColumnFamily` is consumed and should not be used after this call. + /// Drops a column family through a shared reference. /// - /// # Arguments + /// # Safety /// - /// * `cf` - The column family to delete - pub fn delete_column_family(&self, cf: ColumnFamily) -> Result<()> { - let result = unsafe { ffi::tidesdb_delete_column_family(self.db, cf.cf) }; - // Prevent ColumnFamily's Drop from trying to clear commit hook on a deleted CF - std::mem::forget(cf); - check_result(result, "failed to delete column family") + /// No [`ColumnFamily`] handle for `name`, and no transaction or iterator that has + /// used one, may be used after this call returns. + pub unsafe fn drop_column_family_unchecked(&self, name: &str) -> Result<()> { + let c_name = CString::new(name)?; + let result = unsafe { ffi::tidesdb_drop_column_family(self.db, c_name.as_ptr()) }; + check_result(result, "failed to drop column family") } - /// Atomically renames a column family and its underlying directory. - /// Waits for any in-progress flush/compaction to complete before renaming. + /// Renames a column family in place. /// - /// # Arguments - /// - /// * `old_name` - Current name of the column family - /// * `new_name` - New name for the column family + /// Existing handles stay valid and keep referring to the family, but + /// [`ColumnFamily::name`] keeps reporting the name the handle was looked up by. pub fn rename_column_family(&self, old_name: &str, new_name: &str) -> Result<()> { let c_old_name = CString::new(old_name)?; let c_new_name = CString::new(new_name)?; @@ -291,13 +236,9 @@ impl TidesDB { check_result(result, "failed to rename column family") } - /// Creates a complete copy of an existing column family with a new name. - /// The clone contains all the data from the source at the time of cloning. - /// - /// # Arguments + /// Creates a point-in-time copy of a column family under a new name. /// - /// * `source_name` - Name of the source column family to clone - /// * `dest_name` - Name for the new cloned column family + /// Everything written before the call is included; later writes to the source are not. pub fn clone_column_family(&self, source_name: &str, dest_name: &str) -> Result<()> { let c_source_name = CString::new(source_name)?; let c_dest_name = CString::new(dest_name)?; @@ -309,15 +250,7 @@ impl TidesDB { } /// Retrieves a column family by name. - /// - /// # Arguments - /// - /// * `name` - The name of the column family - /// - /// # Returns - /// - /// The column family or an error if not found. - pub fn get_column_family(&self, name: &str) -> Result { + pub fn get_column_family(&self, name: &str) -> Result> { let c_name = CString::new(name)?; let cf = unsafe { ffi::tidesdb_get_column_family(self.db, c_name.as_ptr()) }; @@ -329,36 +262,30 @@ impl TidesDB { } Ok(ColumnFamily { + db: self, cf, name: name.to_string(), - hook_ctx: None, }) } - /// Lists all column families in the database. - /// - /// # Returns - /// - /// A vector of column family names. + /// Lists the names of all column families. pub fn list_column_families(&self) -> Result> { let mut names: *mut *mut c_char = ptr::null_mut(); - let mut count: i32 = 0; + let mut count: c_int = 0; let result = unsafe { ffi::tidesdb_list_column_families(self.db, &mut names, &mut count) }; check_result(result, "failed to list column families")?; - if count == 0 || names.is_null() { + if count <= 0 || names.is_null() { return Ok(Vec::new()); } let mut result_names = Vec::with_capacity(count as usize); - unsafe { - for i in 0..count as isize { - let name_ptr = *names.offset(i); + for i in 0..count as usize { + let name_ptr = *names.add(i); if !name_ptr.is_null() { - let name = CStr::from_ptr(name_ptr).to_string_lossy().into_owned(); - result_names.push(name); + result_names.push(CStr::from_ptr(name_ptr).to_string_lossy().into_owned()); ffi::tidesdb_free(name_ptr as *mut c_void); } } @@ -368,331 +295,265 @@ impl TidesDB { Ok(result_names) } - /// Cancels background compaction work for this database. - /// - /// Flushes are unaffected, so durability is preserved. TidesDB documents - /// this as a fast-shutdown helper intended to be called before close. - #[cfg(tidesdb_has_cancel_background_work)] - pub fn cancel_background_work(&self) -> Result<()> { - let result = unsafe { ffi::tidesdb_cancel_background_work(self.db) }; - check_result(result, "failed to cancel background work") - } - - /// Begins a new transaction with the default isolation level. - /// - /// # Returns - /// - /// A new transaction. - pub fn begin_transaction(&self) -> Result { - let mut txn: *mut ffi::tidesdb_txn_t = ptr::null_mut(); + // ===== transactions ===== - let result = unsafe { ffi::tidesdb_txn_begin(self.db, &mut txn) }; + fn wrap_txn(&self, result: c_int, txn: *mut ffi::tidesdb_txn_t) -> Result> { check_result(result, "failed to begin transaction")?; - if txn.is_null() { return Err(Error::NullPointer("transaction handle")); } + Ok(unsafe { Transaction::from_raw(self.db, txn) }) + } - Ok(Transaction::new(txn)) + /// Begins a transaction at the database default isolation level (read committed). + pub fn begin_transaction(&self) -> Result> { + let mut txn = ptr::null_mut(); + let result = unsafe { ffi::tidesdb_txn_begin(self.db, &mut txn) }; + self.wrap_txn(result, txn) } - /// Begins a new transaction with the specified isolation level. - /// - /// # Arguments - /// - /// * `isolation` - The isolation level for the transaction - /// - /// # Returns - /// - /// A new transaction. + /// Begins a transaction at an explicit isolation level. pub fn begin_transaction_with_isolation( &self, isolation: IsolationLevel, - ) -> Result { - let mut txn: *mut ffi::tidesdb_txn_t = ptr::null_mut(); - + ) -> Result> { + let mut txn = ptr::null_mut(); let result = unsafe { ffi::tidesdb_txn_begin_with_isolation(self.db, isolation as i32, &mut txn) }; - check_result(result, "failed to begin transaction with isolation")?; - - if txn.is_null() { - return Err(Error::NullPointer("transaction handle")); - } + self.wrap_txn(result, txn) + } - Ok(Transaction::new(txn)) + /// Begins a transaction at the column family's configured default isolation level. + pub fn begin_transaction_cf(&self, cf: &ColumnFamily<'_>) -> Result> { + self.check_cf(cf)?; + let mut txn = ptr::null_mut(); + let result = unsafe { ffi::tidesdb_txn_begin_cf(self.db, cf.cf, &mut txn) }; + self.wrap_txn(result, txn) } - /// Retrieves statistics about the block cache. - /// - /// # Returns + /// Takes a snapshot of the database as it stands now. /// - /// Cache statistics. - pub fn get_cache_stats(&self) -> Result { - let mut c_stats = ffi::tidesdb_cache_stats_t { - enabled: 0, - total_entries: 0, - total_bytes: 0, - hits: 0, - misses: 0, - hit_rate: 0.0, - num_partitions: 0, - }; - - let result = unsafe { ffi::tidesdb_get_cache_stats(self.db, &mut c_stats) }; - check_result(result, "failed to get cache stats")?; - - Ok(CacheStats { - enabled: c_stats.enabled != 0, - total_entries: c_stats.total_entries, - total_bytes: c_stats.total_bytes, - hits: c_stats.hits as usize, - misses: c_stats.misses as usize, - hit_rate: c_stats.hit_rate, - num_partitions: c_stats.num_partitions, - }) + /// The snapshot holds back compaction of the versions it can see for as long as it + /// lives, which has the same cost as a long-running transaction; drop it as soon as + /// the point in time is no longer needed. + pub fn snapshot(&self) -> Result> { + let mut snap = ptr::null_mut(); + let result = unsafe { ffi::tidesdb_snapshot_create(self.db, &mut snap) }; + check_result(result, "failed to create snapshot")?; + if snap.is_null() { + return Err(Error::NullPointer("snapshot handle")); + } + Ok(Snapshot { db: self, snap }) } - /// Registers a custom comparator with the database. - /// - /// The comparator function determines the sort order of keys throughout the entire - /// syste, memtables, SSTables, block indexes, and iterators. Once a comparator - /// is set for a column family, it **cannot be changed** without corrupting data. - /// - /// # Arguments - /// - /// * `name` - The comparator name (used in `ColumnFamilyConfig::comparator_name`) - /// * `compare_fn` - A comparison function that returns <0 if key1 < key2, - /// 0 if equal, >0 if key1 > key2 - /// - /// # Example - /// - /// ```no_run - /// use tidesdb::{TidesDB, Config}; - /// - /// let db = TidesDB::open(Config::new("./mydb"))?; + /// Begins a repeatable-read transaction that sees the database as of `snapshot`. /// - /// db.register_comparator("reverse", |key1, key2| { - /// // Reverse byte comparison - /// let min_len = key1.len().min(key2.len()); - /// for i in 0..min_len { - /// if key1[i] != key2[i] { - /// return key2[i] as i32 - key1[i] as i32; - /// } - /// } - /// key2.len() as i32 - key1.len() as i32 - /// })?; - /// # Ok::<(), tidesdb::Error>(()) - /// ``` - pub fn register_comparator(&self, name: &str, compare_fn: F) -> Result<()> - where - F: Fn(&[u8], &[u8]) -> i32 + Send + Sync + 'static, - { - let c_name = CString::new(name)?; - - let boxed: Box = Box::new(Box::new(compare_fn)); - let raw = Box::into_raw(boxed); - - let result = unsafe { - ffi::tidesdb_register_comparator( - self.db, - c_name.as_ptr(), - Some(comparator_trampoline), - std::ptr::null(), // ctx_str - raw as *mut c_void, - ) - }; - - if result != ffi::TDB_SUCCESS { - // Reclaim the box if the C call failed - unsafe { - drop(Box::from_raw(raw)); - } - return Err(Error::from_code(result, "failed to register comparator")); + /// The transaction is not read-only: writes are accepted, and its commit validates its + /// reads against that older point in time. It borrows the snapshot, which must outlive it. + pub fn begin_transaction_at_snapshot<'s>( + &'s self, + snapshot: &'s Snapshot<'s>, + ) -> Result> { + if !ptr::eq(snapshot.db, self) { + return Err(Error::from_code( + ffi::TDB_ERR_INVALID_ARGS, + "snapshot belongs to a different database", + )); } + let mut txn = ptr::null_mut(); + let result = + unsafe { ffi::tidesdb_txn_begin_at_snapshot(self.db, snapshot.snap, &mut txn) }; + self.wrap_txn(result, txn) + } - // The context is now owned by the C library for the lifetime of the database. - // It will be leaked intentionally - the C API has no destroy callback for comparators. - // The memory is freed when the process exits. + /// Begins a repeatable-read transaction that reads as of an explicit sequence number, + /// such as one read back from [`Transaction::read_snapshot`]. Like + /// [`begin_transaction_at_snapshot`](Self::begin_transaction_at_snapshot), it accepts writes. + /// + /// Returns [`ErrorCode::TooOld`](crate::ErrorCode::TooOld) when compaction has already + /// discarded the versions that sequence resolves to. A sequence stays readable only + /// while an open transaction or a [`Snapshot`] holds it. A sequence above the current one + /// is treated as the current one. + pub fn begin_transaction_at_seq(&self, seq: u64) -> Result> { + let mut txn = ptr::null_mut(); + let result = unsafe { ffi::tidesdb_txn_begin_at_seq(self.db, seq, &mut txn) }; + self.wrap_txn(result, txn) + } - Ok(()) + /// The oldest sequence [`begin_transaction_at_seq`](Self::begin_transaction_at_seq) + /// still accepts. It only rises while the database is open; it is not persisted and + /// starts again from 0 after a reopen. + pub fn oldest_readable_seq(&self) -> u64 { + unsafe { ffi::tidesdb_oldest_readable_seq(self.db) } } - /// Checks if a comparator is registered with the database. + /// Lists transactions that were prepared (two-phase commit) before the last shutdown + /// and never committed or rolled back. /// - /// # Arguments + /// Each returned transaction is in the prepared state; resolve it with + /// [`Transaction::commit_prepared`] or [`Transaction::rollback_prepared`]. /// - /// * `name` - The comparator name to look up - /// - /// # Returns - /// - /// `true` if the comparator is registered, `false` otherwise. - pub fn has_comparator(&self, name: &str) -> bool { - let c_name = match CString::new(name) { - Ok(s) => s, - Err(_) => return false, - }; - - let mut fn_out: ffi::tidesdb_comparator_fn = None; - let mut ctx_out: *mut c_void = ptr::null_mut(); + /// The set is fixed when the database opens: calling this again before reopening lists + /// the same transactions, including ones already resolved. Resolve each entry once — + /// committing a stale entry applies its batch again, over any newer writes. + pub fn recover_prepared(&self) -> Result>> { + let mut count: c_int = 0; + let result = + unsafe { ffi::tidesdb_recover_prepared(self.db, ptr::null_mut(), 0, &mut count) }; + check_result(result, "failed to count prepared transactions")?; + if count <= 0 { + return Ok(Vec::new()); + } - let result = unsafe { - ffi::tidesdb_get_comparator(self.db, c_name.as_ptr(), &mut fn_out, &mut ctx_out) - }; + // The set is fixed when the database opens, so the count cannot change between calls. + let mut out: Vec = Vec::with_capacity(count as usize); + let mut filled: c_int = 0; + let result = + unsafe { ffi::tidesdb_recover_prepared(self.db, out.as_mut_ptr(), count, &mut filled) }; + check_result(result, "failed to recover prepared transactions")?; + unsafe { out.set_len(filled.clamp(0, count) as usize) }; + + Ok(out + .into_iter() + .filter(|p| !p.txn.is_null()) + .map(|p| PreparedTransaction { + // the xid is borrowed from the database; copy it out + xid: unsafe { bytes_from_raw(p.xid, p.xid_size) }, + txn: unsafe { Transaction::from_raw(self.db, p.txn) }, + }) + .collect()) + } + + // ===== maintenance ===== + + /// Rotates the memtable and flushes it to sstables, waiting up to 60 seconds for the + /// immutable queue to drain ([`ErrorCode::Locked`](crate::ErrorCode::Locked) if it does + /// not). The memtable is shared, so this flushes every column family. + pub fn flush_memtable(&self) -> Result<()> { + let result = unsafe { ffi::tidesdb_flush_memtable(self.db) }; + check_result(result, "failed to flush memtable") + } - result == ffi::TDB_SUCCESS + /// Whether the memtable is currently flushing or rotating. + pub fn is_flushing(&self) -> bool { + unsafe { ffi::tidesdb_is_flushing(self.db) != 0 } } - /// Creates a backup of the database to the specified directory. - /// - /// # Arguments + /// Forces an fsync of the write-ahead log. /// - /// * `dir` - The directory to backup to - pub fn backup(&self, dir: &str) -> Result<()> { - let c_dir = CString::new(dir)?; + /// Useful for explicit durability control under [`SyncMode::None`](crate::SyncMode::None) + /// or [`SyncMode::Interval`](crate::SyncMode::Interval). + pub fn sync_wal(&self) -> Result<()> { + let result = unsafe { ffi::tidesdb_sync_wal(self.db) }; + check_result(result, "failed to sync WAL") + } - let result = unsafe { ffi::tidesdb_backup(self.db, c_dir.as_ptr() as *mut c_char) }; + /// Writes a consistent, directly openable copy of the database into `dir`, which must + /// be absent or empty (a non-empty directory fails with + /// [`ErrorCode::Io`](crate::ErrorCode::Io)). + pub fn backup>(&self, dir: P) -> Result<()> { + let c_dir = CString::new(dir.as_ref().to_string_lossy().as_bytes())?; + let result = unsafe { ffi::tidesdb_backup(self.db, c_dir.as_ptr()) }; check_result(result, "failed to backup database") } - /// Creates a lightweight, near-instant snapshot of the database using hard links - /// instead of copying SSTable data. - /// - /// # Arguments - /// - /// * `checkpoint_dir` - The directory to create the checkpoint in. - /// Must be a non-existent or empty directory. - pub fn checkpoint(&self, checkpoint_dir: &str) -> Result<()> { - let c_dir = CString::new(checkpoint_dir)?; - - let result = unsafe { ffi::tidesdb_checkpoint(self.db, c_dir.as_ptr()) }; + /// Establishes a durability barrier: flushes the memtable to L1, then forces the value + /// log, write-ahead log and manifest to disk regardless of the configured sync mode. + pub fn checkpoint(&self) -> Result<()> { + let result = unsafe { ffi::tidesdb_checkpoint(self.db) }; check_result(result, "failed to checkpoint database") } - /// Forces a synchronous flush and aggressive compaction for **all** column families, - /// then drains both the global flush and compaction queues. - /// - /// This is a blocking operation - it will not return until all flush and compaction - /// work is complete. - pub fn purge(&self) -> Result<()> { - let result = unsafe { ffi::tidesdb_purge(self.db) }; - check_result(result, "failed to purge database") - } + // ===== statistics ===== - /// Retrieves aggregate statistics across the entire database instance. - /// - /// Unlike `ColumnFamily::get_stats` (which heap-allocates), this fills a - /// caller-provided struct on the stack. No free is needed. + /// Database-level statistics. pub fn get_db_stats(&self) -> Result { - let mut c_stats = std::mem::MaybeUninit::::zeroed(); - + let mut c_stats = MaybeUninit::::zeroed(); let result = unsafe { ffi::tidesdb_get_db_stats(self.db, c_stats.as_mut_ptr()) }; check_result(result, "failed to get database stats")?; + Ok(DbStats::from_c(unsafe { c_stats.assume_init_ref() })) + } - let c_stats = unsafe { c_stats.assume_init() }; - - // Write-amplification counters (tidesdb >= 9.3.4; 0 on older libraries). - #[cfg(tidesdb_has_write_amp_stats)] - let ( - uwal_bytes_written, - wal_bytes_written, - flush_bytes_written, - compaction_bytes_written, - compaction_bytes_read, - user_bytes_written, - flush_count, - compaction_count, - ) = ( - c_stats.uwal_bytes_written, - c_stats.wal_bytes_written, - c_stats.flush_bytes_written, - c_stats.compaction_bytes_written, - c_stats.compaction_bytes_read, - c_stats.user_bytes_written, - c_stats.flush_count, - c_stats.compaction_count, - ); - #[cfg(not(tidesdb_has_write_amp_stats))] - let ( - uwal_bytes_written, - wal_bytes_written, - flush_bytes_written, - compaction_bytes_written, - compaction_bytes_read, - user_bytes_written, - flush_count, - compaction_count, - ) = (0u64, 0u64, 0u64, 0u64, 0u64, 0u64, 0u64, 0u64); - - // Single-writer fencing epochs (tidesdb >= 9.3.8; 0 on older libraries). - #[cfg(tidesdb_has_single_writer_fencing)] - let (primary_epoch, seen_epoch) = (c_stats.primary_epoch, c_stats.seen_epoch); - #[cfg(not(tidesdb_has_single_writer_fencing))] - let (primary_epoch, seen_epoch) = (0u64, 0u64); - - let object_store_connector = if c_stats.object_store_connector.is_null() { - String::new() - } else { - unsafe { CStr::from_ptr(c_stats.object_store_connector) } - .to_string_lossy() - .into_owned() - }; - - Ok(DbStats { - num_column_families: c_stats.num_column_families, - total_memory: c_stats.total_memory, - available_memory: c_stats.available_memory, - resolved_memory_limit: c_stats.resolved_memory_limit, - memory_pressure_level: c_stats.memory_pressure_level, - flush_pending_count: c_stats.flush_pending_count, - total_memtable_bytes: c_stats.total_memtable_bytes, - total_immutable_count: c_stats.total_immutable_count, - total_sstable_count: c_stats.total_sstable_count, - total_data_size_bytes: c_stats.total_data_size_bytes, - num_open_sstables: c_stats.num_open_sstables, - global_seq: c_stats.global_seq, - txn_memory_bytes: c_stats.txn_memory_bytes, - compaction_queue_size: c_stats.compaction_queue_size, - flush_queue_size: c_stats.flush_queue_size, - unified_memtable_enabled: c_stats.unified_memtable_enabled != 0, - unified_memtable_bytes: c_stats.unified_memtable_bytes, - unified_immutable_count: c_stats.unified_immutable_count, - unified_is_flushing: c_stats.unified_is_flushing != 0, - unified_next_cf_index: c_stats.unified_next_cf_index, - unified_wal_generation: c_stats.unified_wal_generation, - object_store_enabled: c_stats.object_store_enabled != 0, - object_store_connector, - local_cache_bytes_used: c_stats.local_cache_bytes_used, - local_cache_bytes_max: c_stats.local_cache_bytes_max, - local_cache_num_files: c_stats.local_cache_num_files, - last_uploaded_generation: c_stats.last_uploaded_generation, - upload_queue_depth: c_stats.upload_queue_depth, - total_uploads: c_stats.total_uploads, - total_upload_failures: c_stats.total_upload_failures, - replica_mode: c_stats.replica_mode != 0, - primary_epoch, - seen_epoch, - uwal_bytes_written, - wal_bytes_written, - flush_bytes_written, - compaction_bytes_written, - compaction_bytes_read, - user_bytes_written, - flush_count, - compaction_count, + /// Block cache statistics. + pub fn get_cache_stats(&self) -> Result { + let mut c_stats = MaybeUninit::::zeroed(); + let result = unsafe { ffi::tidesdb_get_cache_stats(self.db, c_stats.as_mut_ptr()) }; + check_result(result, "failed to get cache stats")?; + let c = unsafe { c_stats.assume_init() }; + Ok(CacheStats { + enabled: c.enabled != 0, + total_entries: c.total_entries, + total_bytes: c.total_bytes, + hits: c.hits, + misses: c.misses, + hit_rate: c.hit_rate, + num_partitions: c.num_partitions, }) } - /// Switches a read-only replica to primary mode. - /// - /// This is only valid when the database was opened in replica mode - /// (via object store configuration with `replica_mode` enabled). - /// - /// # Returns - /// - /// `Ok(())` on success, or an error if the database is not in replica mode. - pub fn promote_to_primary(&self) -> Result<()> { - let result = unsafe { ffi::tidesdb_promote_to_primary(self.db) }; - check_result(result, "failed to promote to primary") + /// Where writers have been made to wait since the database opened. + pub fn get_stall_stats(&self) -> Result { + let mut c_stats = MaybeUninit::::zeroed(); + let result = unsafe { ffi::tidesdb_get_stall_stats(self.db, c_stats.as_mut_ptr()) }; + check_result(result, "failed to get stall stats")?; + Ok(StallStats::from_c(unsafe { c_stats.assume_init_ref() })) + } + + /// What each class of file asked of the device since the database opened. + pub fn get_io_stats(&self) -> Result { + let mut c_stats = MaybeUninit::::zeroed(); + let result = unsafe { ffi::tidesdb_get_io_stats(self.db, c_stats.as_mut_ptr()) }; + check_result(result, "failed to get I/O stats")?; + Ok(IoStats::from_c(unsafe { c_stats.assume_init_ref() })) + } + + /// What each encoding chain achieved on the key logs, one entry per chain the live + /// sstables were written with. + pub fn get_klog_encoding_stats(&self) -> Result> { + self.encoding_stats(ffi::tidesdb_get_klog_encoding_stats) + } + + /// What each encoding chain achieved on the separated values in the value log. + pub fn get_vlog_encoding_stats(&self) -> Result> { + self.encoding_stats(ffi::tidesdb_get_vlog_encoding_stats) + } + + fn encoding_stats( + &self, + f: unsafe extern "C" fn( + *mut ffi::tidesdb_t, + *mut ffi::tidesdb_encoding_stats_t, + usize, + *mut usize, + ) -> c_int, + ) -> Result> { + let mut out = + [MaybeUninit::::zeroed(); ffi::TDB_MAX_ENCODING_CHAINS]; + let mut count: usize = 0; + let result = unsafe { + f( + self.db, + out.as_mut_ptr() as *mut ffi::tidesdb_encoding_stats_t, + out.len(), + &mut count, + ) + }; + check_result(result, "failed to get encoding stats")?; + Ok(out[..count.min(out.len())] + .iter() + .map(|e| EncodingStats::from_c(unsafe { e.assume_init_ref() })) + .collect()) + } + + fn check_cf(&self, cf: &ColumnFamily<'_>) -> Result<()> { + if ptr::eq(cf.db, self) { + Ok(()) + } else { + Err(Error::from_code( + ffi::TDB_ERR_INVALID_ARGS, + "column family belongs to a different database", + )) + } } } @@ -704,419 +565,202 @@ impl Drop for TidesDB { } self.db = ptr::null_mut(); } + // The hooks are freed by the Mutex's own drop, after close has joined every + // engine thread, so no commit can still be running one. } } -/// A column family in TidesDB. +/// A column family: an isolated key space with its own configuration. /// -/// Column families are isolated key-value stores with independent configuration. -pub struct ColumnFamily { +/// Handles are cheap to clone and borrow the [`TidesDB`] they came from. +#[derive(Clone)] +pub struct ColumnFamily<'db> { + pub(crate) db: &'db TidesDB, pub(crate) cf: *mut ffi::tidesdb_column_family_t, name: String, - /// Stored commit hook context for cleanup on drop/clear. - hook_ctx: Option<*mut CommitHookCallback>, } -// ColumnFamily uses internal locking for thread safety -unsafe impl Send for ColumnFamily {} -unsafe impl Sync for ColumnFamily {} +// The C library synchronises column family operations internally. +unsafe impl Send for ColumnFamily<'_> {} +unsafe impl Sync for ColumnFamily<'_> {} -impl ColumnFamily { - /// Gets the name of this column family. +impl std::fmt::Debug for ColumnFamily<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ColumnFamily") + .field("name", &self.name) + .finish() + } +} + +impl<'db> ColumnFamily<'db> { + /// The name this handle was looked up by. After a rename it still reports the old name; + /// [`CfStats::name`](crate::CfStats::name) from [`get_stats`](Self::get_stats) is the + /// current one. pub fn name(&self) -> &str { &self.name } - /// Retrieves statistics about this column family. - pub fn get_stats(&self) -> Result { - let mut c_stats: *mut ffi::tidesdb_stats_t = ptr::null_mut(); - - let result = unsafe { ffi::tidesdb_get_stats(self.cf, &mut c_stats) }; - check_result(result, "failed to get stats")?; - - if c_stats.is_null() { - return Err(Error::NullPointer("stats")); - } - - let stats = unsafe { - let num_levels = (*c_stats).num_levels; - let memtable_size = (*c_stats).memtable_size; - - let mut level_sizes = Vec::new(); - if num_levels > 0 && !(*c_stats).level_sizes.is_null() { - for i in 0..num_levels as isize { - level_sizes.push(*(*c_stats).level_sizes.offset(i)); - } - } - - let mut level_num_sstables = Vec::new(); - if num_levels > 0 && !(*c_stats).level_num_sstables.is_null() { - for i in 0..num_levels as isize { - level_num_sstables.push(*(*c_stats).level_num_sstables.offset(i)); - } - } + /// Statistics for this column family. + pub fn get_stats(&self) -> Result { + // boxed: the struct is over 600 bytes and filled in place + let mut c_stats = Box::new(MaybeUninit::::zeroed()); + let result = unsafe { ffi::tidesdb_get_cf_stats(self.cf, c_stats.as_mut_ptr()) }; + check_result(result, "failed to get column family stats")?; + Ok(CfStats::from_c(unsafe { c_stats.assume_init_ref() })) + } - let mut level_key_counts = Vec::new(); - if num_levels > 0 && !(*c_stats).level_key_counts.is_null() { - for i in 0..num_levels as isize { - level_key_counts.push(*(*c_stats).level_key_counts.offset(i)); - } - } + /// Estimates the number of distinct keys in this column family. + pub fn estimate_cardinality(&self) -> Result { + let mut estimate: u64 = 0; + let result = unsafe { ffi::tidesdb_cf_estimate_cardinality(self.cf, &mut estimate) }; + check_result(result, "failed to estimate cardinality")?; + Ok(estimate) + } - let level_tombstone_counts = { - #[cfg(tidesdb_has_tombstone_stats)] - { - let mut counts = Vec::new(); - if num_levels > 0 && !(*c_stats).level_tombstone_counts.is_null() { - for i in 0..num_levels as isize { - counts.push(*(*c_stats).level_tombstone_counts.offset(i)); - } - } - counts - } - #[cfg(not(tidesdb_has_tombstone_stats))] - { - Vec::new() - } - }; - - let total_keys = (*c_stats).total_keys; - let total_data_size = (*c_stats).total_data_size; - let avg_key_size = (*c_stats).avg_key_size; - let avg_value_size = (*c_stats).avg_value_size; - let read_amp = (*c_stats).read_amp; - let hit_rate = (*c_stats).hit_rate; - let use_btree = (*c_stats).use_btree != 0; - let btree_total_nodes = (*c_stats).btree_total_nodes; - let btree_max_height = (*c_stats).btree_max_height; - let btree_avg_height = (*c_stats).btree_avg_height; - let total_tombstones = { - #[cfg(tidesdb_has_tombstone_stats)] - { - (*c_stats).total_tombstones - } - #[cfg(not(tidesdb_has_tombstone_stats))] - { - 0 - } - }; - let tombstone_ratio = { - #[cfg(tidesdb_has_tombstone_stats)] - { - (*c_stats).tombstone_ratio - } - #[cfg(not(tidesdb_has_tombstone_stats))] - { - 0.0 - } - }; - let max_sst_density = { - #[cfg(tidesdb_has_tombstone_stats)] - { - (*c_stats).max_sst_density - } - #[cfg(not(tidesdb_has_tombstone_stats))] - { - 0.0 - } - }; - let max_sst_density_level = { - #[cfg(tidesdb_has_tombstone_stats)] - { - (*c_stats).max_sst_density_level - } - #[cfg(not(tidesdb_has_tombstone_stats))] - { - 0 - } - }; - - // Write-amplification counters (tidesdb >= 9.3.4; 0 on older libraries). - #[cfg(tidesdb_has_write_amp_stats)] - let ( - wal_bytes_written, - flush_bytes_written, - compaction_bytes_written, - compaction_bytes_read, - user_bytes_written, - flush_count, - compaction_count, - ) = ( - (*c_stats).wal_bytes_written, - (*c_stats).flush_bytes_written, - (*c_stats).compaction_bytes_written, - (*c_stats).compaction_bytes_read, - (*c_stats).user_bytes_written, - (*c_stats).flush_count, - (*c_stats).compaction_count, - ); - #[cfg(not(tidesdb_has_write_amp_stats))] - let ( - wal_bytes_written, - flush_bytes_written, - compaction_bytes_written, - compaction_bytes_read, - user_bytes_written, - flush_count, - compaction_count, - ) = (0u64, 0u64, 0u64, 0u64, 0u64, 0u64, 0u64); - - let config = if (*c_stats).config.is_null() { - None - } else { - Some(ColumnFamilyConfig::from_c_config_ptr((*c_stats).config)) - }; - - ffi::tidesdb_free_stats(c_stats); - - crate::stats::Stats { - num_levels, - memtable_size, - level_sizes, - level_num_sstables, - config, - total_keys, - total_data_size, - avg_key_size, - avg_value_size, - level_key_counts, - read_amp, - hit_rate, - use_btree, - btree_total_nodes, - btree_max_height, - btree_avg_height, - total_tombstones, - tombstone_ratio, - level_tombstone_counts, - max_sst_density, - max_sst_density_level, - wal_bytes_written, - flush_bytes_written, - compaction_bytes_written, - compaction_bytes_read, - user_bytes_written, - flush_count, - compaction_count, - } + /// Describes the key range `[key_a, key_b)` for a query planner: how many sorted runs a + /// scan would merge and how many live keys it holds. + pub fn range_stats(&self, key_a: &[u8], key_b: &[u8]) -> Result { + let mut out = MaybeUninit::::zeroed(); + let result = unsafe { + ffi::tidesdb_range_stats( + self.db.db, + self.cf, + key_a.as_ptr(), + key_a.len(), + key_b.as_ptr(), + key_b.len(), + out.as_mut_ptr(), + ) }; - - Ok(stats) + check_result(result, "failed to get range stats")?; + let c = unsafe { out.assume_init() }; + Ok(RangeStats { + sstables_overlapping: c.sstables_overlapping, + estimated_keys: c.estimated_keys, + keys_exact: c.keys_exact != 0, + }) } - /// Manually triggers compaction for this column family. + /// Synchronously runs one forced compaction pass, merging even when no trigger is due. + /// + /// Returns [`ErrorCode::Locked`](crate::ErrorCode::Locked) without waiting if the family + /// is already compacting, being reconfigured or being renamed. pub fn compact(&self) -> Result<()> { - let result = unsafe { ffi::tidesdb_compact(self.cf) }; + let result = unsafe { ffi::tidesdb_compact(self.db.db, self.cf) }; check_result(result, "failed to compact column family") } - /// Synchronously compacts every SSTable whose key range overlaps `[start_key, end_key)`. - /// - /// Blocks the calling thread until the merge commits or fails (it does not enqueue - /// onto the compaction thread pool). `None` means unbounded on that side; passing - /// `None` for both endpoints is rejected -- use [`compact`](Self::compact) for full - /// column-family compaction. - /// - /// # Arguments - /// - /// * `start_key` - Inclusive start of the range, or `None` for unbounded - /// * `end_key` - Exclusive end of the range, or `None` for unbounded + /// Synchronously compacts every sstable overlapping `[start_key, end_key)`. /// - /// # Returns - /// - /// `Ok(())` on success, or an error: `InvalidArgs` if both endpoints are `None`, - /// `Locked` if another compaction is already running, or standard I/O / memory errors. - #[cfg(tidesdb_has_compact_range)] + /// `None` is unbounded on that side; passing `None` for both is rejected, use + /// [`compact`](Self::compact) instead. Returns [`ErrorCode::Locked`](crate::ErrorCode::Locked) + /// like `compact`, and does nothing when fewer than two sstables overlap the range. pub fn compact_range(&self, start_key: Option<&[u8]>, end_key: Option<&[u8]>) -> Result<()> { - // Empty slices are treated as unbounded (NULL) to avoid passing a - // dangling pointer-of-element-zero deref to C. - let (start_ptr, start_len) = match start_key { - Some(s) if !s.is_empty() => (s.as_ptr(), s.len()), - _ => (std::ptr::null(), 0), - }; - let (end_ptr, end_len) = match end_key { - Some(s) if !s.is_empty() => (s.as_ptr(), s.len()), - _ => (std::ptr::null(), 0), - }; + let (start_ptr, start_len) = start_key.map_or((ptr::null(), 0), |s| (s.as_ptr(), s.len())); + let (end_ptr, end_len) = end_key.map_or((ptr::null(), 0), |s| (s.as_ptr(), s.len())); - let result = - unsafe { ffi::tidesdb_compact_range(self.cf, start_ptr, start_len, end_ptr, end_len) }; + let result = unsafe { + ffi::tidesdb_compact_range(self.db.db, self.cf, start_ptr, start_len, end_ptr, end_len) + }; check_result(result, "failed to compact range") } - /// Manually triggers memtable flush for this column family. - pub fn flush_memtable(&self) -> Result<()> { - let result = unsafe { ffi::tidesdb_flush_memtable(self.cf) }; - check_result(result, "failed to flush memtable") - } - - /// Checks if this column family has a flush operation in progress. - pub fn is_flushing(&self) -> bool { - unsafe { ffi::tidesdb_is_flushing(self.cf) != 0 } - } - - /// Checks if this column family has a compaction operation in progress. + /// Whether this column family has a compaction in progress. pub fn is_compacting(&self) -> bool { unsafe { ffi::tidesdb_is_compacting(self.cf) != 0 } } - /// Forces a synchronous flush and aggressive compaction for this column family. - /// - /// Unlike `flush_memtable` and `compact` (which are non-blocking), purge blocks - /// until all flush and compaction I/O is complete. - /// - /// **Behavior:** - /// 1. Waits for any in-progress flush to complete - /// 2. Force-flushes the active memtable (even if below threshold) - /// 3. Waits for flush I/O to fully complete - /// 4. Waits for any in-progress compaction to complete - /// 5. Triggers synchronous compaction inline (bypasses the compaction queue) - /// 6. Waits for any queued compaction to drain - pub fn purge(&self) -> Result<()> { - let result = unsafe { ffi::tidesdb_purge_cf(self.cf) }; - check_result(result, "failed to purge column family") - } - - /// Forces an immediate fsync of the active write-ahead log for this column family. - /// - /// This is useful for explicit durability control when using `SyncMode::None` or - /// `SyncMode::Interval`. - /// - /// **When to use:** - /// - Application-controlled durability after a batch of related writes - /// - Pre-checkpoint to ensure all buffered WAL data is on disk - /// - Graceful shutdown to flush WAL buffers before closing - /// - Critical writes that need durability without `SyncMode::Full` for all writes - pub fn sync_wal(&self) -> Result<()> { - let result = unsafe { ffi::tidesdb_sync_wal(self.cf) }; - check_result(result, "failed to sync WAL") - } - - /// Estimates the computational cost of iterating between two keys in this column family. - /// The returned value is an opaque double - meaningful only for comparison with other - /// values from the same function. Uses only in-memory metadata and performs no disk I/O. - /// - /// Key order does not matter - the function normalizes the range so `key_a > key_b` - /// produces the same result as `key_b > key_a`. + /// Applies a new configuration at runtime. The name is preserved. /// - /// # Arguments - /// - /// * `key_a` - First key (bound of range) - /// * `key_b` - Second key (bound of range) - /// - /// # Returns - /// - /// Estimated traversal cost (higher = more expensive). A cost of 0.0 means no - /// overlapping SSTables or memtable entries were found for the range. - pub fn range_cost(&self, key_a: &[u8], key_b: &[u8]) -> Result { - let mut cost: f64 = 0.0; - + /// With `persist_to_disk` the new configuration is written to the manifest; otherwise + /// it lasts until the database is reopened (a later rename persists it too). Returns + /// [`ErrorCode::Locked`](crate::ErrorCode::Locked) without waiting if the family is + /// compacting or under another reconfiguration. + pub fn update_runtime_config( + &self, + config: &ColumnFamilyConfig, + persist_to_disk: bool, + ) -> Result<()> { + let c_config = config.to_c_config()?; let result = unsafe { - ffi::tidesdb_range_cost( + ffi::tidesdb_cf_update_runtime_config( + self.db.db, self.cf, - key_a.as_ptr(), - key_a.len(), - key_b.as_ptr(), - key_b.len(), - &mut cost, + &c_config, + persist_to_disk as c_int, ) }; - check_result(result, "failed to estimate range cost")?; - - Ok(cost) + check_result(result, "failed to update runtime config") } - /// Sets a commit hook callback for this column family. - /// - /// The hook fires synchronously after every transaction commit on this column family. - /// It receives the full batch of committed operations atomically, enabling real-time - /// change data capture without WAL parsing. - /// - /// The hook fires after WAL write, memtable apply, and commit status marking are - /// complete - the data is fully durable before the callback runs. Hook failure - /// (non-zero return) is logged but does not affect the commit result. + /// Installs a commit hook for this column family, replacing any existing one. /// - /// # Arguments + /// The hook fires synchronously on the committing thread after every commit that + /// touches this family, with the family's whole batch, once the commit is logged, + /// applied and visible (not necessarily synced to disk, depending on the sync mode). + /// Commits on different threads may run it concurrently, so calls can arrive out of + /// `commit_seq` order. A range or prefix delete arrives as one delete op keyed by its + /// lower bound. A non-zero return is logged by the engine but does not undo the commit; + /// a panic is caught and counts as a failure. /// - /// * `callback` - A closure receiving a slice of `CommitOp` and the monotonic - /// commit sequence number. Return 0 on success, non-zero on failure. - pub fn set_commit_hook(&mut self, callback: F) -> Result<()> + /// The hook belongs to the column family, not to this handle: it stays installed until + /// cleared or the database closes. Replaced hooks are kept alive until close. + pub fn set_commit_hook(&self, callback: F) -> Result<()> where - F: Fn(&[CommitOp], u64) -> i32 + Send + 'static, + F: Fn(&[CommitOp], u64) -> i32 + Send + Sync + 'static, { - // Clear any existing hook first - self.clear_commit_hook()?; - let boxed: Box = Box::new(Box::new(callback)); - let raw = Box::into_raw(boxed); + let ctx = &*boxed as *const CommitHookCallback as *mut c_void; + // Hold the lock across the swap so the retained list never misses a live hook. + let mut hooks = self.db.hooks.lock().unwrap_or_else(|e| e.into_inner()); let result = unsafe { - ffi::tidesdb_cf_set_commit_hook( - self.cf, - Some(commit_hook_trampoline), - raw as *mut c_void, - ) + ffi::tidesdb_cf_set_commit_hook(self.db.db, self.cf, Some(commit_hook_trampoline), ctx) }; - - if result != ffi::TDB_SUCCESS { - // Reclaim the box if the C call failed - unsafe { - drop(Box::from_raw(raw)); - } - return Err(Error::from_code(result, "failed to set commit hook")); - } - - self.hook_ctx = Some(raw); + check_result(result, "failed to set commit hook")?; + hooks.push(boxed); Ok(()) } - /// Clears the commit hook for this column family. - /// - /// After clearing, no callback will fire on subsequent commits. - pub fn clear_commit_hook(&mut self) -> Result<()> { - if let Some(raw) = self.hook_ctx.take() { - let result = unsafe { ffi::tidesdb_cf_set_commit_hook(self.cf, None, ptr::null_mut()) }; - // Free the boxed callback regardless of C call result - unsafe { - drop(Box::from_raw(raw)); - } - check_result(result, "failed to clear commit hook")?; - } - Ok(()) + /// Removes this column family's commit hook. + pub fn clear_commit_hook(&self) -> Result<()> { + let result = + unsafe { ffi::tidesdb_cf_set_commit_hook(self.db.db, self.cf, None, ptr::null_mut()) }; + check_result(result, "failed to clear commit hook") } +} - /// Updates the runtime configuration for this column family. - /// - /// # Arguments - /// - /// * `config` - The new configuration - /// * `persist_to_disk` - Whether to persist the configuration to disk - pub fn update_runtime_config( - &self, - config: &ColumnFamilyConfig, - persist_to_disk: bool, - ) -> Result<()> { - let c_config = config.to_c_config(); - let result = unsafe { - ffi::tidesdb_cf_update_runtime_config( - self.cf, - &c_config, - if persist_to_disk { 1 } else { 0 }, - ) - }; - check_result(result, "failed to update runtime config") +/// A named point in time that transactions can read at. +/// +/// Created by [`TidesDB::snapshot`] and released when dropped. While it lives, the versions +/// it can see are protected from compaction. +pub struct Snapshot<'db> { + db: &'db TidesDB, + snap: *mut ffi::tidesdb_snapshot_t, +} + +// A snapshot is only read after creation; beginning transactions at it from several +// threads is safe. +unsafe impl Send for Snapshot<'_> {} +unsafe impl Sync for Snapshot<'_> {} + +impl Snapshot<'_> { + /// The sequence number this snapshot reads at. + pub fn seq(&self) -> u64 { + unsafe { ffi::tidesdb_snapshot_seq(self.snap) } } } -impl Drop for ColumnFamily { +impl Drop for Snapshot<'_> { fn drop(&mut self) { - // Clear the commit hook to free the boxed callback. - // We ignore the result since we're in Drop. - if let Some(raw) = self.hook_ctx.take() { - unsafe { - let _ = ffi::tidesdb_cf_set_commit_hook(self.cf, None, ptr::null_mut()); - drop(Box::from_raw(raw)); - } - } + unsafe { ffi::tidesdb_snapshot_release(self.snap) }; } } + +/// A transaction recovered by [`TidesDB::recover_prepared`]. +pub struct PreparedTransaction<'db> { + /// The transaction id it was prepared under + pub xid: Vec, + /// The transaction, in the prepared state + pub txn: Transaction<'db>, +} diff --git a/src/error.rs b/src/error.rs index f7de17a..a951079 100644 --- a/src/error.rs +++ b/src/error.rs @@ -6,11 +6,15 @@ //! Error types for TidesDB operations. use crate::ffi; +use std::ffi::CStr; use std::fmt; use thiserror::Error; /// Error codes returned by TidesDB operations. +/// +/// New codes may be added in minor releases, so match with a wildcard arm. #[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] pub enum ErrorCode { /// Memory allocation failed Memory, @@ -24,68 +28,85 @@ pub enum ErrorCode { Corruption, /// Resource already exists Exists, - /// Transaction conflict + /// Transaction conflict: another transaction committed a conflicting write first Conflict, - /// Key or value too large + /// A value does not fit the space that has to hold it TooLarge, /// Memory limit exceeded MemoryLimit, - /// Invalid database handle + /// Invalid database handle, or the database is closing InvalidDb, /// Unknown error Unknown, - /// Database is locked + /// Transient contention. Nothing was written and the operation should be retried; + /// it never means absence or data loss. Locked, /// Database is in read-only mode ReadOnly, - /// Resource is busy (another operation is in progress) - Busy, - /// A required precondition was not met - Precondition, + /// The transaction outlived its timeout and was aborted + TxnExpired, + /// The device or filesystem is out of space; the operation succeeds once space is freed + NoSpace, + /// The transaction was aborted from another thread with + /// [`AbortHandle::request_abort`](crate::AbortHandle::request_abort) + TxnAborted, + /// The requested point in time is no longer reconstructable + TooOld, } impl ErrorCode { - /// Convert from a C error code - pub fn from_code(code: i32) -> Option { + /// Convert from a C error code. Unrecognised codes map to [`ErrorCode::Unknown`]. + pub fn from_code(code: i32) -> Self { match code { - ffi::TDB_ERR_MEMORY => Some(ErrorCode::Memory), - ffi::TDB_ERR_INVALID_ARGS => Some(ErrorCode::InvalidArgs), - ffi::TDB_ERR_NOT_FOUND => Some(ErrorCode::NotFound), - ffi::TDB_ERR_IO => Some(ErrorCode::Io), - ffi::TDB_ERR_CORRUPTION => Some(ErrorCode::Corruption), - ffi::TDB_ERR_EXISTS => Some(ErrorCode::Exists), - ffi::TDB_ERR_CONFLICT => Some(ErrorCode::Conflict), - ffi::TDB_ERR_TOO_LARGE => Some(ErrorCode::TooLarge), - ffi::TDB_ERR_MEMORY_LIMIT => Some(ErrorCode::MemoryLimit), - ffi::TDB_ERR_INVALID_DB => Some(ErrorCode::InvalidDb), - ffi::TDB_ERR_LOCKED => Some(ErrorCode::Locked), - ffi::TDB_ERR_READONLY => Some(ErrorCode::ReadOnly), - ffi::TDB_ERR_BUSY => Some(ErrorCode::Busy), - ffi::TDB_ERR_PRECONDITION => Some(ErrorCode::Precondition), - _ => Some(ErrorCode::Unknown), + ffi::TDB_ERR_MEMORY => ErrorCode::Memory, + ffi::TDB_ERR_INVALID_ARGS => ErrorCode::InvalidArgs, + ffi::TDB_ERR_NOT_FOUND => ErrorCode::NotFound, + ffi::TDB_ERR_IO => ErrorCode::Io, + ffi::TDB_ERR_CORRUPTION => ErrorCode::Corruption, + ffi::TDB_ERR_EXISTS => ErrorCode::Exists, + ffi::TDB_ERR_CONFLICT => ErrorCode::Conflict, + ffi::TDB_ERR_TOO_LARGE => ErrorCode::TooLarge, + ffi::TDB_ERR_MEMORY_LIMIT => ErrorCode::MemoryLimit, + ffi::TDB_ERR_INVALID_DB => ErrorCode::InvalidDb, + ffi::TDB_ERR_LOCKED => ErrorCode::Locked, + ffi::TDB_ERR_READONLY => ErrorCode::ReadOnly, + ffi::TDB_ERR_TXN_EXPIRED => ErrorCode::TxnExpired, + ffi::TDB_ERR_NO_SPACE => ErrorCode::NoSpace, + ffi::TDB_ERR_TXN_ABORTED => ErrorCode::TxnAborted, + ffi::TDB_ERR_TOO_OLD => ErrorCode::TooOld, + _ => ErrorCode::Unknown, + } + } + + /// The C error code (`TDB_ERR_*`) this variant corresponds to. + pub fn code(self) -> i32 { + match self { + ErrorCode::Memory => ffi::TDB_ERR_MEMORY, + ErrorCode::InvalidArgs => ffi::TDB_ERR_INVALID_ARGS, + ErrorCode::NotFound => ffi::TDB_ERR_NOT_FOUND, + ErrorCode::Io => ffi::TDB_ERR_IO, + ErrorCode::Corruption => ffi::TDB_ERR_CORRUPTION, + ErrorCode::Exists => ffi::TDB_ERR_EXISTS, + ErrorCode::Conflict => ffi::TDB_ERR_CONFLICT, + ErrorCode::TooLarge => ffi::TDB_ERR_TOO_LARGE, + ErrorCode::MemoryLimit => ffi::TDB_ERR_MEMORY_LIMIT, + ErrorCode::InvalidDb => ffi::TDB_ERR_INVALID_DB, + ErrorCode::Unknown => ffi::TDB_ERR_UNKNOWN, + ErrorCode::Locked => ffi::TDB_ERR_LOCKED, + ErrorCode::ReadOnly => ffi::TDB_ERR_READONLY, + ErrorCode::TxnExpired => ffi::TDB_ERR_TXN_EXPIRED, + ErrorCode::NoSpace => ffi::TDB_ERR_NO_SPACE, + ErrorCode::TxnAborted => ffi::TDB_ERR_TXN_ABORTED, + ErrorCode::TooOld => ffi::TDB_ERR_TOO_OLD, } } } impl fmt::Display for ErrorCode { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - ErrorCode::Memory => write!(f, "memory allocation failed"), - ErrorCode::InvalidArgs => write!(f, "invalid arguments"), - ErrorCode::NotFound => write!(f, "not found"), - ErrorCode::Io => write!(f, "I/O error"), - ErrorCode::Corruption => write!(f, "data corruption"), - ErrorCode::Exists => write!(f, "already exists"), - ErrorCode::Conflict => write!(f, "transaction conflict"), - ErrorCode::TooLarge => write!(f, "key or value too large"), - ErrorCode::MemoryLimit => write!(f, "memory limit exceeded"), - ErrorCode::InvalidDb => write!(f, "invalid database handle"), - ErrorCode::Unknown => write!(f, "unknown error"), - ErrorCode::Locked => write!(f, "database is locked"), - ErrorCode::ReadOnly => write!(f, "database is read-only"), - ErrorCode::Busy => write!(f, "resource is busy"), - ErrorCode::Precondition => write!(f, "precondition not met"), - } + // tidesdb_strerror returns a static literal, never NULL + let msg = unsafe { CStr::from_ptr(ffi::tidesdb_strerror(self.code())) }; + f.write_str(&msg.to_string_lossy()) } } @@ -116,31 +137,32 @@ impl Error { /// Create a new TidesDB error from a C error code pub fn from_code(code: i32, context: &'static str) -> Self { Error::TidesDB { - code: ErrorCode::from_code(code).unwrap_or(ErrorCode::Unknown), + code: ErrorCode::from_code(code), context, } } + /// The TidesDB error code, if this error came from the C library + pub fn code(&self) -> Option { + match self { + Error::TidesDB { code, .. } => Some(*code), + _ => None, + } + } + /// Check if this is a "not found" error pub fn is_not_found(&self) -> bool { - matches!( - self, - Error::TidesDB { - code: ErrorCode::NotFound, - .. - } - ) + self.code() == Some(ErrorCode::NotFound) } - /// Check if this is a conflict error + /// Check if this is a transaction conflict error pub fn is_conflict(&self) -> bool { - matches!( - self, - Error::TidesDB { - code: ErrorCode::Conflict, - .. - } - ) + self.code() == Some(ErrorCode::Conflict) + } + + /// Check if this is transient contention ([`ErrorCode::Locked`]) that should be retried + pub fn is_locked(&self) -> bool { + self.code() == Some(ErrorCode::Locked) } } diff --git a/src/ffi.rs b/src/ffi.rs index 0cf005d..31f8ef4 100644 --- a/src/ffi.rs +++ b/src/ffi.rs @@ -4,20 +4,73 @@ // Licensed under the Mozilla Public License, v. 2.0 (the "License"); //! Raw FFI bindings to the TidesDB C library. +//! +//! Mirrors `include/db.h` of the bundled TidesDB release (see `TIDESDB_VERSION`). +//! Every `#[repr(C)]` struct here must match the C layout exactly, since several +//! are filled in place by the library. The layout tests at the bottom of this file +//! pin the sizes so a header change cannot slip through silently. +//! //! These are unsafe and should not be used directly. #![allow(dead_code)] +#![allow(non_camel_case_types)] -use libc::{c_char, c_double, c_float, c_int, c_void, size_t, time_t}; +use libc::{c_char, c_double, c_float, c_int, c_long, c_void, size_t, time_t}; -/// Maximum comparator name length -pub const TDB_MAX_COMPARATOR_NAME: usize = 64; -/// Maximum comparator context string length -pub const TDB_MAX_COMPARATOR_CTX: usize = 256; -/// Maximum column family name length -pub const TDB_MAX_CF_NAME_LEN: usize = 128; +/// The TidesDB release these bindings were written against. +pub const TIDESDB_VERSION: &str = "10.2.1"; + +// ===== enums ===== + +/// Log levels (`tidesdb_log_level_t`) +pub const TDB_LOG_NONE: c_int = 0; +pub const TDB_LOG_TRACE: c_int = 1; +pub const TDB_LOG_INFO: c_int = 2; +pub const TDB_LOG_WARN: c_int = 3; +pub const TDB_LOG_ERROR: c_int = 4; + +/// Isolation levels (`tidesdb_isolation_level_t`) +pub const TDB_ISOLATION_READ_UNCOMMITTED: c_int = 0; +pub const TDB_ISOLATION_READ_COMMITTED: c_int = 1; +pub const TDB_ISOLATION_REPEATABLE_READ: c_int = 2; +pub const TDB_ISOLATION_SNAPSHOT: c_int = 3; +pub const TDB_ISOLATION_SERIALIZABLE: c_int = 4; + +/// Transaction lifecycle states (`tidesdb_txn_state_t`) +pub const TDB_TXN_STATE_ACTIVE: c_int = 0; +pub const TDB_TXN_STATE_PREPARED: c_int = 1; +pub const TDB_TXN_STATE_COMMITTED: c_int = 2; +pub const TDB_TXN_STATE_ABORTED: c_int = 3; + +/// Compression algorithms (`tidesdb_compression_algorithm_t`), also the encoding ids +/// accepted in a column family's encoding pipeline. +pub const TDB_COMPRESS_NONE: c_int = 0; +pub const TDB_COMPRESS_SNAPPY: c_int = 1; +pub const TDB_COMPRESS_LZ4: c_int = 2; +pub const TDB_COMPRESS_ZSTD: c_int = 3; +pub const TDB_COMPRESS_LZ4_FAST: c_int = 4; + +/// Sync modes (`tidesdb_sync_mode_t`) +pub const TDB_SYNC_NONE: c_int = 0; +pub const TDB_SYNC_FULL: c_int = 1; +pub const TDB_SYNC_INTERVAL: c_int = 2; + +/// Stall reasons (`tidesdb_stall_reason_t`) +pub const TDB_STALL_WAL_APPEND: c_int = 0; +pub const TDB_STALL_ROTATE_LOCK: c_int = 1; +pub const TDB_STALL_ROTATE_WORK: c_int = 2; +pub const TDB_STALL_ADMISSION: c_int = 3; +pub const TDB_STALL_MANIFEST_COMMIT: c_int = 4; +pub const TDB_STALL_COUNT: usize = 5; + +/// I/O classes (`tidesdb_io_class_t`) +pub const TDB_IO_SSTABLE: c_int = 0; +pub const TDB_IO_WAL: c_int = 1; +pub const TDB_IO_VLOG: c_int = 2; +pub const TDB_IO_COUNT: usize = 3; + +// ===== error codes ===== -/// Error codes (negative values as per C API) pub const TDB_SUCCESS: c_int = 0; pub const TDB_ERR_MEMORY: c_int = -1; pub const TDB_ERR_INVALID_ARGS: c_int = -2; @@ -32,49 +85,23 @@ pub const TDB_ERR_INVALID_DB: c_int = -10; pub const TDB_ERR_UNKNOWN: c_int = -11; pub const TDB_ERR_LOCKED: c_int = -12; pub const TDB_ERR_READONLY: c_int = -13; -pub const TDB_ERR_BUSY: c_int = -14; -pub const TDB_ERR_PRECONDITION: c_int = -15; - -/// Object store backend identifiers (`tidesdb_objstore_backend_t`) -pub const TDB_BACKEND_FS: c_int = 0; -pub const TDB_BACKEND_S3: c_int = 1; -pub const TDB_BACKEND_UNKNOWN: c_int = 99; - -/// Compression algorithms (values from C API documentation) -pub const NO_COMPRESSION: c_int = 0; -pub const SNAPPY_COMPRESSION: c_int = 1; -pub const LZ4_COMPRESSION: c_int = 2; -pub const ZSTD_COMPRESSION: c_int = 3; -pub const LZ4_FAST_COMPRESSION: c_int = 4; - -/// Sync modes -pub const TDB_SYNC_NONE: c_int = 0; -pub const TDB_SYNC_FULL: c_int = 1; -pub const TDB_SYNC_INTERVAL: c_int = 2; +pub const TDB_ERR_TXN_EXPIRED: c_int = -14; +pub const TDB_ERR_NO_SPACE: c_int = -15; +pub const TDB_ERR_TXN_ABORTED: c_int = -16; +pub const TDB_ERR_TOO_OLD: c_int = -17; -/// Log levels -pub const TDB_LOG_DEBUG: c_int = 0; -pub const TDB_LOG_INFO: c_int = 1; -pub const TDB_LOG_WARN: c_int = 2; -pub const TDB_LOG_ERROR: c_int = 3; -pub const TDB_LOG_FATAL: c_int = 4; -pub const TDB_LOG_NONE: c_int = 99; +// ===== configuration limits ===== -/// Isolation levels -pub const TDB_ISOLATION_READ_UNCOMMITTED: c_int = 0; -pub const TDB_ISOLATION_READ_COMMITTED: c_int = 1; -pub const TDB_ISOLATION_REPEATABLE_READ: c_int = 2; -pub const TDB_ISOLATION_SNAPSHOT: c_int = 3; -pub const TDB_ISOLATION_SERIALIZABLE: c_int = 4; +pub const TDB_MAX_CF_NAME_LEN: usize = 128; +pub const TDB_ENCODING_PIPELINE_MAX: usize = 8; +pub const TDB_MAX_RANGE_BOUND_SIZE: usize = 256; +pub const TDB_MAX_LEVELS: usize = 8; +pub const TDB_MAX_ENCODING_CHAINS: usize = 16; -/// Opaque types -#[repr(C)] -pub struct tidesdb_t { - _private: [u8; 0], -} +// ===== opaque handles ===== #[repr(C)] -pub struct tidesdb_txn_t { +pub struct tidesdb_t { _private: [u8; 0], } @@ -84,106 +111,23 @@ pub struct tidesdb_column_family_t { } #[repr(C)] -pub struct tidesdb_iter_t { +pub struct tidesdb_txn_t { _private: [u8; 0], } #[repr(C)] -pub struct tidesdb_objstore_t { +pub struct tidesdb_snapshot_t { _private: [u8; 0], } -/// Object store behavior configuration -#[repr(C)] -pub struct tidesdb_objstore_config_t { - pub local_cache_path: *const c_char, - pub local_cache_max_bytes: size_t, - pub cache_on_read: c_int, - pub cache_on_write: c_int, - pub max_concurrent_uploads: c_int, - pub max_concurrent_downloads: c_int, - pub multipart_threshold: size_t, - pub multipart_part_size: size_t, - pub sync_manifest_to_object: c_int, - pub replicate_wal: c_int, - pub wal_upload_sync: c_int, - pub wal_sync_threshold_bytes: size_t, - pub wal_sync_on_commit: c_int, - pub replica_mode: c_int, - pub replica_sync_interval_us: u64, - pub replica_replay_wal: c_int, -} - -/// Full S3 connector configuration (TLS + multipart tuning). -/// -/// Zero-initialize and set what you need; the all-zero defaults are secure -/// (TLS verify on, no custom CA) and use the built-in multipart sizes. #[repr(C)] -pub struct tidesdb_objstore_s3_config_t { - pub endpoint: *const c_char, - pub bucket: *const c_char, - pub prefix: *const c_char, - pub access_key: *const c_char, - pub secret_key: *const c_char, - pub region: *const c_char, - pub use_ssl: c_int, - pub use_path_style: c_int, - pub tls_ca_path: *const c_char, - pub tls_insecure_skip_verify: c_int, - pub multipart_threshold: size_t, - pub multipart_part_size: size_t, +pub struct tidesdb_iter_t { + _private: [u8; 0], } -/// Custom allocator function types -#[allow(non_camel_case_types)] -pub type tidesdb_malloc_fn = Option *mut c_void>; -#[allow(non_camel_case_types)] -pub type tidesdb_calloc_fn = - Option *mut c_void>; -#[allow(non_camel_case_types)] -pub type tidesdb_realloc_fn = - Option *mut c_void>; -#[allow(non_camel_case_types)] -pub type tidesdb_free_fn = Option; - -/// Comparator function type -#[allow(non_camel_case_types)] -pub type tidesdb_comparator_fn = Option< - unsafe extern "C" fn( - key1: *const u8, - key1_size: size_t, - key2: *const u8, - key2_size: size_t, - ctx: *mut c_void, - ) -> c_int, ->; - -/// Database configuration -#[repr(C)] -pub struct tidesdb_config_t { - pub db_path: *const c_char, - pub num_flush_threads: c_int, - pub num_compaction_threads: c_int, - pub log_level: c_int, - pub block_cache_size: size_t, - pub max_open_sstables: size_t, - pub log_to_file: c_int, - pub log_truncation_at: size_t, - pub max_memory_usage: size_t, - pub unified_memtable: c_int, - pub unified_memtable_write_buffer_size: size_t, - pub unified_memtable_skip_list_max_level: c_int, - pub unified_memtable_skip_list_probability: c_float, - pub unified_memtable_sync_mode: c_int, - pub unified_memtable_sync_interval_us: u64, - pub object_store: *mut tidesdb_objstore_t, - pub object_store_config: *mut tidesdb_objstore_config_t, - // Added in TidesDB 9.2.0. Older 9.0.x/9.1.x tags have a shorter C struct. - #[cfg(tidesdb_has_max_concurrent_flushes)] - pub max_concurrent_flushes: c_int, -} +// ===== callback types ===== -/// Commit operation passed to commit hook callback +/// One operation in a committed transaction batch, passed to the commit hook #[repr(C)] pub struct tidesdb_commit_op_t { pub key: *const u8, @@ -195,7 +139,6 @@ pub struct tidesdb_commit_op_t { } /// Commit hook callback function type -#[allow(non_camel_case_types)] pub type tidesdb_commit_hook_fn = Option< unsafe extern "C" fn( ops: *const tidesdb_commit_op_t, @@ -205,192 +148,234 @@ pub type tidesdb_commit_hook_fn = Option< ) -> c_int, >; -/// Column family configuration +// ===== configuration ===== + +/// Per-column-family configuration #[repr(C)] +#[derive(Clone, Copy)] pub struct tidesdb_column_family_config_t { pub name: [c_char; TDB_MAX_CF_NAME_LEN], - pub write_buffer_size: size_t, pub level_size_ratio: size_t, pub min_levels: c_int, pub dividing_level_offset: c_int, - pub klog_value_threshold: size_t, - pub compression_algo: c_int, + pub keep_values_inline: c_int, + pub btree_klog_block_size: size_t, + pub encoding_pipeline: [u8; TDB_ENCODING_PIPELINE_MAX], + pub encoding_count: u8, pub enable_bloom_filter: c_int, pub bloom_fpr: c_double, - pub enable_block_indexes: c_int, - pub index_sample_ratio: c_int, - pub block_index_prefix_len: c_int, - pub sync_mode: c_int, - pub sync_interval_us: u64, - pub comparator_name: [c_char; TDB_MAX_COMPARATOR_NAME], - pub comparator_ctx_str: [c_char; TDB_MAX_COMPARATOR_CTX], - pub comparator_fn_cached: *mut c_void, - pub comparator_ctx_cached: *mut c_void, - pub skip_list_max_level: c_int, - pub skip_list_probability: c_float, pub default_isolation_level: c_int, - pub min_disk_space: u64, pub l1_file_count_trigger: c_int, - pub l0_queue_stall_threshold: c_int, - // Added in TidesDB 9.2.0 with tombstone-density compaction controls. - #[cfg(tidesdb_has_tombstone_stats)] pub tombstone_density_trigger: c_double, - #[cfg(tidesdb_has_tombstone_stats)] pub tombstone_density_min_entries: u64, - pub use_btree: c_int, pub commit_hook_fn: tidesdb_commit_hook_fn, pub commit_hook_ctx: *mut c_void, - /// Reserved field; preserved for ABI compatibility with the C struct. - pub object_target_file_size: size_t, - pub object_lazy_compaction: c_int, - pub object_prefetch_compaction: c_int, } -/// Statistics for a column family +/// Database-level configuration #[repr(C)] -pub struct tidesdb_stats_t { +pub struct tidesdb_config_t { + pub db_path: *mut c_char, + pub num_flush_threads: c_int, + pub num_compaction_threads: c_int, + pub log_level: c_int, + pub block_cache_size: size_t, + pub max_open_sstables: size_t, + pub log_to_file: c_int, + pub log_truncation_at: size_t, + pub memtable_write_buffer_size: size_t, + pub memtable_skip_list_max_level: c_int, + pub memtable_skip_list_probability: c_float, + pub memtable_sync_mode: c_int, + pub memtable_sync_interval_us: u64, + pub value_separation_threshold: size_t, + pub vlog_segment_size: size_t, + pub memtable_l0_queue_stall_threshold: c_int, + pub memtable_idle_flush_seconds: c_int, + pub txn_timeout_seconds: i64, +} + +// ===== statistics ===== + +/// Per-column-family statistics, a flat value with nothing to free +#[repr(C)] +pub struct tidesdb_cf_stats_t { pub num_levels: c_int, - pub memtable_size: size_t, - pub level_sizes: *mut size_t, - pub level_num_sstables: *mut c_int, - pub config: *mut tidesdb_column_family_config_t, + pub config: tidesdb_column_family_config_t, + pub level_sizes: [size_t; TDB_MAX_LEVELS], + pub level_num_sstables: [c_int; TDB_MAX_LEVELS], + pub level_key_counts: [u64; TDB_MAX_LEVELS], + pub level_tombstone_counts: [u64; TDB_MAX_LEVELS], pub total_keys: u64, pub total_data_size: u64, pub avg_key_size: c_double, pub avg_value_size: c_double, - pub level_key_counts: *mut u64, pub read_amp: c_double, - pub hit_rate: c_double, - pub use_btree: c_int, pub btree_total_nodes: u64, pub btree_max_height: u32, pub btree_avg_height: c_double, - // Added in TidesDB 9.2.0 with tombstone observability. - #[cfg(tidesdb_has_tombstone_stats)] pub total_tombstones: u64, - #[cfg(tidesdb_has_tombstone_stats)] pub tombstone_ratio: c_double, - #[cfg(tidesdb_has_tombstone_stats)] - pub level_tombstone_counts: *mut u64, - #[cfg(tidesdb_has_tombstone_stats)] pub max_sst_density: c_double, - #[cfg(tidesdb_has_tombstone_stats)] pub max_sst_density_level: c_int, - // Added in TidesDB 9.3.4 with write-amplification observability. - #[cfg(tidesdb_has_write_amp_stats)] pub wal_bytes_written: u64, - #[cfg(tidesdb_has_write_amp_stats)] pub flush_bytes_written: u64, - #[cfg(tidesdb_has_write_amp_stats)] pub compaction_bytes_written: u64, - #[cfg(tidesdb_has_write_amp_stats)] pub compaction_bytes_read: u64, - #[cfg(tidesdb_has_write_amp_stats)] pub user_bytes_written: u64, - #[cfg(tidesdb_has_write_amp_stats)] - pub flush_count: u64, - #[cfg(tidesdb_has_write_amp_stats)] pub compaction_count: u64, + pub unflushed_key_count: u64, + pub filter_resident_bytes: u64, +} + +/// Block cache statistics +#[repr(C)] +pub struct tidesdb_cache_stats_t { + pub enabled: c_int, + pub total_entries: size_t, + pub total_bytes: size_t, + pub hits: u64, + pub misses: u64, + pub hit_rate: c_double, + pub num_partitions: size_t, } -/// Database-level aggregate statistics +/// What one encoding chain achieved on the data it wrote +#[repr(C)] +#[derive(Clone, Copy)] +pub struct tidesdb_encoding_stats_t { + pub ids: [u8; TDB_ENCODING_PIPELINE_MAX], + pub id_count: c_int, + pub logical_bytes: u64, + pub stored_bytes: u64, + pub item_count: u64, +} + +/// Database-level statistics #[repr(C)] pub struct tidesdb_db_stats_t { pub num_column_families: c_int, - pub total_memory: u64, - pub available_memory: u64, - pub resolved_memory_limit: size_t, - pub memory_pressure_level: c_int, - pub flush_pending_count: c_int, - pub total_memtable_bytes: i64, - pub total_immutable_count: c_int, + pub immutable_memtable_count: c_int, + pub compaction_pending_count: c_int, pub total_sstable_count: c_int, pub total_data_size_bytes: u64, pub num_open_sstables: c_int, pub global_seq: u64, + pub min_snapshot_seq: u64, + pub active_txn_count: c_int, pub txn_memory_bytes: i64, - pub compaction_queue_size: size_t, - pub flush_queue_size: size_t, - pub unified_memtable_enabled: c_int, - pub unified_memtable_bytes: i64, - pub unified_immutable_count: c_int, - pub unified_is_flushing: c_int, - pub unified_next_cf_index: u32, - pub unified_wal_generation: u64, - pub object_store_enabled: c_int, - pub object_store_connector: *const c_char, - pub local_cache_bytes_used: size_t, - pub local_cache_bytes_max: size_t, - pub local_cache_num_files: c_int, - pub last_uploaded_generation: u64, - pub upload_queue_depth: size_t, - pub total_uploads: u64, - pub total_upload_failures: u64, - pub replica_mode: c_int, - // Added in TidesDB 9.3.8: single-writer fencing epochs (object-store mode). - // These sit between `replica_mode` and the write-amp counters in the C - // struct, so they must be gated and ordered to match the C layout exactly. - #[cfg(tidesdb_has_single_writer_fencing)] - pub primary_epoch: u64, - #[cfg(tidesdb_has_single_writer_fencing)] - pub seen_epoch: u64, - // Added in TidesDB 9.3.4. This struct is stack-allocated by Rust before - // passing it to C, so the tail must match C exactly to avoid overwrites. - #[cfg(tidesdb_has_write_amp_stats)] - pub uwal_bytes_written: u64, - #[cfg(tidesdb_has_write_amp_stats)] - pub wal_bytes_written: u64, - #[cfg(tidesdb_has_write_amp_stats)] + pub txn_commits: u64, + pub txn_conflicts: u64, + pub memtable_bytes: size_t, + pub is_flushing: c_int, + pub next_cf_index: u32, + pub wal_generation: u64, + pub flush_count: u64, + pub compaction_count: u64, pub flush_bytes_written: u64, - #[cfg(tidesdb_has_write_amp_stats)] pub compaction_bytes_written: u64, - #[cfg(tidesdb_has_write_amp_stats)] pub compaction_bytes_read: u64, - #[cfg(tidesdb_has_write_amp_stats)] + pub wal_bytes_written: u64, pub user_bytes_written: u64, - #[cfg(tidesdb_has_write_amp_stats)] - pub flush_count: u64, - #[cfg(tidesdb_has_write_amp_stats)] - pub compaction_count: u64, + pub vlog_file_size: u64, + pub vlog_value_count: u64, + pub vlog_used_bytes: u64, + pub vlog_stored_bytes: u64, + pub vlog_live_bytes: u64, + pub vlog_segment_count: u64, + pub vlog_bytes_written: u64, + pub vlog_dead_bytes: u64, + pub vlog_reclaim_calls: u64, + pub vlog_reclaim_passes: u64, + pub vlog_segments_retired: u64, + pub vlog_segments_drainable: u64, + pub writes_throttled: u64, + pub writes_blocked: u64, + pub write_stall_us: u64, + pub write_stall_ceiling_hits: u64, } -/// Cache statistics +/// How much waiting one stall reason accounted for #[repr(C)] -pub struct tidesdb_cache_stats_t { - pub enabled: c_int, - pub total_entries: size_t, - pub total_bytes: size_t, - pub hits: u64, - pub misses: u64, - pub hit_rate: c_double, - pub num_partitions: size_t, +#[derive(Clone, Copy)] +pub struct tidesdb_stall_stat_t { + pub count: u64, + pub total_us: u64, + pub max_us: u64, +} + +/// Every stall reason, indexed by `tidesdb_stall_reason_t` +#[repr(C)] +pub struct tidesdb_stall_stats_t { + pub reasons: [tidesdb_stall_stat_t; TDB_STALL_COUNT], +} + +/// What one class of file asked of the device +#[repr(C)] +#[derive(Clone, Copy)] +pub struct tidesdb_io_stat_t { + pub ops: u64, + pub bytes: u64, + pub total_us: u64, + pub max_us: u64, +} + +/// Every I/O class, indexed by `tidesdb_io_class_t` +#[repr(C)] +pub struct tidesdb_io_stats_t { + pub classes: [tidesdb_io_stat_t; TDB_IO_COUNT], +} + +/// Query-planner statistics for a key range +#[repr(C)] +pub struct tidesdb_range_stats_t { + pub sstables_overlapping: u64, + pub estimated_keys: u64, + pub keys_exact: c_int, +} + +/// A transaction durably prepared before a restart and still awaiting a decision +#[repr(C)] +pub struct tidesdb_prepared_txn_t { + pub txn: *mut tidesdb_txn_t, + pub xid: *const u8, + pub xid_size: size_t, } #[link(name = "tidesdb")] unsafe extern "C" { - // Initialization / custom allocator - pub fn tidesdb_init( - malloc_fn: tidesdb_malloc_fn, - calloc_fn: tidesdb_calloc_fn, - realloc_fn: tidesdb_realloc_fn, - free_fn: tidesdb_free_fn, - ) -> c_int; - pub fn tidesdb_finalize(); + // Utilities + pub fn tidesdb_compression_available(algo: c_int) -> c_int; + pub fn tidesdb_strerror(code: c_int) -> *const c_char; + pub fn tidesdb_free(ptr: *mut c_void); + + // Defaults + pub fn tidesdb_default_column_family_config() -> tidesdb_column_family_config_t; + pub fn tidesdb_default_config() -> tidesdb_config_t; + + // Process limits + pub fn tidesdb_raise_open_file_limit(desired: c_long) -> c_long; - // Database operations + // Database lifecycle pub fn tidesdb_open(config: *const tidesdb_config_t, db: *mut *mut tidesdb_t) -> c_int; pub fn tidesdb_close(db: *mut tidesdb_t) -> c_int; - // Column family operations + // Column families pub fn tidesdb_create_column_family( db: *mut tidesdb_t, name: *const c_char, config: *const tidesdb_column_family_config_t, ) -> c_int; pub fn tidesdb_drop_column_family(db: *mut tidesdb_t, name: *const c_char) -> c_int; - pub fn tidesdb_delete_column_family( + pub fn tidesdb_rename_column_family( db: *mut tidesdb_t, - cf: *mut tidesdb_column_family_t, + old_name: *const c_char, + new_name: *const c_char, + ) -> c_int; + pub fn tidesdb_clone_column_family( + db: *mut tidesdb_t, + src_name: *const c_char, + dst_name: *const c_char, ) -> c_int; pub fn tidesdb_get_column_family( db: *mut tidesdb_t, @@ -401,84 +386,135 @@ unsafe extern "C" { names: *mut *mut *mut c_char, count: *mut c_int, ) -> c_int; - - // Default configuration - pub fn tidesdb_default_column_family_config() -> tidesdb_column_family_config_t; - - // Statistics - pub fn tidesdb_get_stats( - cf: *mut tidesdb_column_family_t, - stats: *mut *mut tidesdb_stats_t, - ) -> c_int; - pub fn tidesdb_free_stats(stats: *mut tidesdb_stats_t); - pub fn tidesdb_get_cache_stats(db: *mut tidesdb_t, stats: *mut tidesdb_cache_stats_t) -> c_int; - - // Maintenance - pub fn tidesdb_compact(cf: *mut tidesdb_column_family_t) -> c_int; - #[cfg(tidesdb_has_cancel_background_work)] - pub fn tidesdb_cancel_background_work(db: *mut tidesdb_t) -> c_int; - #[cfg(tidesdb_has_compact_range)] - pub fn tidesdb_compact_range( + pub fn tidesdb_cf_update_runtime_config( + db: *mut tidesdb_t, cf: *mut tidesdb_column_family_t, - start_key: *const u8, - start_key_size: size_t, - end_key: *const u8, - end_key_size: size_t, + new_config: *const tidesdb_column_family_config_t, + persist_to_disk: c_int, ) -> c_int; - pub fn tidesdb_flush_memtable(cf: *mut tidesdb_column_family_t) -> c_int; - #[cfg(tidesdb_has_raise_open_file_limit)] - pub fn tidesdb_raise_open_file_limit(desired: libc::c_long) -> libc::c_long; - - // Comparator - pub fn tidesdb_register_comparator( + pub fn tidesdb_cf_set_commit_hook( db: *mut tidesdb_t, - name: *const c_char, - compare_fn: tidesdb_comparator_fn, - ctx_str: *const c_char, + cf: *mut tidesdb_column_family_t, + hook_fn: tidesdb_commit_hook_fn, ctx: *mut c_void, ) -> c_int; - // Transaction operations + // Transactions pub fn tidesdb_txn_begin(db: *mut tidesdb_t, txn: *mut *mut tidesdb_txn_t) -> c_int; pub fn tidesdb_txn_begin_with_isolation( db: *mut tidesdb_t, isolation: c_int, txn: *mut *mut tidesdb_txn_t, ) -> c_int; + pub fn tidesdb_txn_begin_cf( + db: *mut tidesdb_t, + cf: *mut tidesdb_column_family_t, + txn: *mut *mut tidesdb_txn_t, + ) -> c_int; + + // Snapshots and point-in-time reads + pub fn tidesdb_snapshot_create( + db: *mut tidesdb_t, + snapshot: *mut *mut tidesdb_snapshot_t, + ) -> c_int; + pub fn tidesdb_snapshot_release(snapshot: *mut tidesdb_snapshot_t); + pub fn tidesdb_snapshot_seq(snapshot: *const tidesdb_snapshot_t) -> u64; + pub fn tidesdb_txn_begin_at_snapshot( + db: *mut tidesdb_t, + snapshot: *mut tidesdb_snapshot_t, + txn: *mut *mut tidesdb_txn_t, + ) -> c_int; + pub fn tidesdb_txn_begin_at_seq( + db: *mut tidesdb_t, + seq: u64, + txn: *mut *mut tidesdb_txn_t, + ) -> c_int; + pub fn tidesdb_oldest_readable_seq(db: *const tidesdb_t) -> u64; + + // Transaction operations pub fn tidesdb_txn_put( txn: *mut tidesdb_txn_t, cf: *mut tidesdb_column_family_t, key: *const u8, - key_len: size_t, + key_size: size_t, value: *const u8, - value_len: size_t, + value_size: size_t, ttl: time_t, ) -> c_int; pub fn tidesdb_txn_get( txn: *mut tidesdb_txn_t, cf: *mut tidesdb_column_family_t, key: *const u8, - key_len: size_t, + key_size: size_t, value: *mut *mut u8, - value_len: *mut size_t, + value_size: *mut size_t, + ) -> c_int; + pub fn tidesdb_txn_get_ttl( + txn: *mut tidesdb_txn_t, + cf: *mut tidesdb_column_family_t, + key: *const u8, + key_size: size_t, + ttl: *mut time_t, + ) -> c_int; + pub fn tidesdb_txn_get_notrack( + txn: *mut tidesdb_txn_t, + cf: *mut tidesdb_column_family_t, + key: *const u8, + key_size: size_t, + value: *mut *mut u8, + value_size: *mut size_t, + ) -> c_int; + pub fn tidesdb_txn_read_snapshot(txn: *const tidesdb_txn_t) -> u64; + pub fn tidesdb_txn_contains( + txn: *mut tidesdb_txn_t, + cf: *mut tidesdb_column_family_t, + key: *const u8, + key_size: size_t, + ) -> c_int; + pub fn tidesdb_txn_delete_range( + txn: *mut tidesdb_txn_t, + cf: *mut tidesdb_column_family_t, + lo: *const u8, + lo_size: size_t, + hi: *const u8, + hi_size: size_t, + ) -> c_int; + pub fn tidesdb_txn_delete_prefix( + txn: *mut tidesdb_txn_t, + cf: *mut tidesdb_column_family_t, + prefix: *const u8, + prefix_size: size_t, ) -> c_int; pub fn tidesdb_txn_delete( txn: *mut tidesdb_txn_t, cf: *mut tidesdb_column_family_t, key: *const u8, - key_len: size_t, + key_size: size_t, ) -> c_int; - #[cfg(tidesdb_has_txn_single_delete)] pub fn tidesdb_txn_single_delete( txn: *mut tidesdb_txn_t, cf: *mut tidesdb_column_family_t, key: *const u8, - key_len: size_t, + key_size: size_t, ) -> c_int; pub fn tidesdb_txn_commit(txn: *mut tidesdb_txn_t) -> c_int; pub fn tidesdb_txn_rollback(txn: *mut tidesdb_txn_t) -> c_int; - pub fn tidesdb_txn_free(txn: *mut tidesdb_txn_t); + pub fn tidesdb_txn_set_timeout(txn: *mut tidesdb_txn_t, seconds: i64) -> c_int; + pub fn tidesdb_txn_request_abort(txn: *mut tidesdb_txn_t); pub fn tidesdb_txn_reset(txn: *mut tidesdb_txn_t, isolation: c_int) -> c_int; + pub fn tidesdb_txn_free(txn: *mut tidesdb_txn_t); + + // Two-phase commit + pub fn tidesdb_txn_prepare(txn: *mut tidesdb_txn_t, xid: *const u8, xid_size: size_t) -> c_int; + pub fn tidesdb_txn_commit_prepared(txn: *mut tidesdb_txn_t) -> c_int; + pub fn tidesdb_txn_rollback_prepared(txn: *mut tidesdb_txn_t) -> c_int; + pub fn tidesdb_recover_prepared( + db: *mut tidesdb_t, + out: *mut tidesdb_prepared_txn_t, + max: c_int, + out_count: *mut c_int, + ) -> c_int; + pub fn tidesdb_txn_state(txn: *const tidesdb_txn_t, out_state: *mut c_int) -> c_int; // Savepoints pub fn tidesdb_txn_savepoint(txn: *mut tidesdb_txn_t, name: *const c_char) -> c_int; @@ -486,202 +522,139 @@ unsafe extern "C" { -> c_int; pub fn tidesdb_txn_release_savepoint(txn: *mut tidesdb_txn_t, name: *const c_char) -> c_int; - // Iterator operations + // Iterators pub fn tidesdb_iter_new( txn: *mut tidesdb_txn_t, cf: *mut tidesdb_column_family_t, iter: *mut *mut tidesdb_iter_t, ) -> c_int; - pub fn tidesdb_iter_seek_to_first(iter: *mut tidesdb_iter_t) -> c_int; - pub fn tidesdb_iter_seek_to_last(iter: *mut tidesdb_iter_t) -> c_int; - pub fn tidesdb_iter_seek(iter: *mut tidesdb_iter_t, key: *const u8, key_len: size_t) -> c_int; + pub fn tidesdb_iter_new_range( + txn: *mut tidesdb_txn_t, + cf: *mut tidesdb_column_family_t, + lower: *const u8, + lower_size: size_t, + upper: *const u8, + upper_size: size_t, + iter: *mut *mut tidesdb_iter_t, + ) -> c_int; + pub fn tidesdb_iter_seek(iter: *mut tidesdb_iter_t, key: *const u8, key_size: size_t) -> c_int; pub fn tidesdb_iter_seek_for_prev( iter: *mut tidesdb_iter_t, key: *const u8, - key_len: size_t, + key_size: size_t, ) -> c_int; - pub fn tidesdb_iter_valid(iter: *mut tidesdb_iter_t) -> c_int; + pub fn tidesdb_iter_seek_to_first(iter: *mut tidesdb_iter_t) -> c_int; + pub fn tidesdb_iter_seek_to_last(iter: *mut tidesdb_iter_t) -> c_int; pub fn tidesdb_iter_next(iter: *mut tidesdb_iter_t) -> c_int; pub fn tidesdb_iter_prev(iter: *mut tidesdb_iter_t) -> c_int; + pub fn tidesdb_iter_valid(iter: *mut tidesdb_iter_t) -> c_int; pub fn tidesdb_iter_key( iter: *mut tidesdb_iter_t, key: *mut *mut u8, - key_len: *mut size_t, + key_size: *mut size_t, ) -> c_int; pub fn tidesdb_iter_value( iter: *mut tidesdb_iter_t, value: *mut *mut u8, - value_len: *mut size_t, + value_size: *mut size_t, ) -> c_int; - pub fn tidesdb_iter_free(iter: *mut tidesdb_iter_t); - - // Column family rename - pub fn tidesdb_rename_column_family( - db: *mut tidesdb_t, - old_name: *const c_char, - new_name: *const c_char, + pub fn tidesdb_iter_ttl(iter: *mut tidesdb_iter_t, ttl: *mut time_t) -> c_int; + pub fn tidesdb_iter_key_value( + iter: *mut tidesdb_iter_t, + key: *mut *mut u8, + key_size: *mut size_t, + value: *mut *mut u8, + value_size: *mut size_t, ) -> c_int; + pub fn tidesdb_iter_free(iter: *mut tidesdb_iter_t); - // Column family clone - pub fn tidesdb_clone_column_family( + // Maintenance + pub fn tidesdb_compact(db: *mut tidesdb_t, cf: *mut tidesdb_column_family_t) -> c_int; + pub fn tidesdb_compact_range( db: *mut tidesdb_t, - source_name: *const c_char, - dest_name: *const c_char, + cf: *mut tidesdb_column_family_t, + start_key: *const u8, + start_key_size: size_t, + end_key: *const u8, + end_key_size: size_t, ) -> c_int; - - // Default config - pub fn tidesdb_default_config() -> tidesdb_config_t; - - // Backup - pub fn tidesdb_backup(db: *mut tidesdb_t, dir: *mut c_char) -> c_int; - - // Checkpoint - pub fn tidesdb_checkpoint(db: *mut tidesdb_t, checkpoint_dir: *const c_char) -> c_int; - - // Flushing/compacting status - pub fn tidesdb_is_flushing(cf: *mut tidesdb_column_family_t) -> c_int; + pub fn tidesdb_flush_memtable(db: *mut tidesdb_t) -> c_int; + pub fn tidesdb_is_flushing(db: *mut tidesdb_t) -> c_int; pub fn tidesdb_is_compacting(cf: *mut tidesdb_column_family_t) -> c_int; + pub fn tidesdb_sync_wal(db: *mut tidesdb_t) -> c_int; + pub fn tidesdb_backup(db: *mut tidesdb_t, dir: *const c_char) -> c_int; + pub fn tidesdb_checkpoint(db: *mut tidesdb_t) -> c_int; - // Config INI operations - pub fn tidesdb_cf_config_load_from_ini( - ini_file: *const c_char, - section_name: *const c_char, - config: *mut tidesdb_column_family_config_t, - ) -> c_int; - pub fn tidesdb_cf_config_save_to_ini( - ini_file: *const c_char, - section_name: *const c_char, - config: *const tidesdb_column_family_config_t, + // Statistics + pub fn tidesdb_get_cf_stats( + cf: *mut tidesdb_column_family_t, + stats: *mut tidesdb_cf_stats_t, ) -> c_int; - - // Runtime config update - pub fn tidesdb_cf_update_runtime_config( + pub fn tidesdb_cf_estimate_cardinality( cf: *mut tidesdb_column_family_t, - new_config: *const tidesdb_column_family_config_t, - persist_to_disk: c_int, + out_estimate: *mut u64, ) -> c_int; - - // Get comparator - pub fn tidesdb_get_comparator( + pub fn tidesdb_get_db_stats(db: *mut tidesdb_t, stats: *mut tidesdb_db_stats_t) -> c_int; + pub fn tidesdb_get_klog_encoding_stats( db: *mut tidesdb_t, - name: *const c_char, - fn_out: *mut tidesdb_comparator_fn, - ctx_out: *mut *mut c_void, + out: *mut tidesdb_encoding_stats_t, + max: size_t, + out_count: *mut size_t, ) -> c_int; - - // Commit hook - pub fn tidesdb_cf_set_commit_hook( - cf: *mut tidesdb_column_family_t, - hook_fn: tidesdb_commit_hook_fn, - ctx: *mut c_void, + pub fn tidesdb_get_vlog_encoding_stats( + db: *mut tidesdb_t, + out: *mut tidesdb_encoding_stats_t, + max: size_t, + out_count: *mut size_t, ) -> c_int; - - // Range cost estimation - pub fn tidesdb_range_cost( + pub fn tidesdb_get_cache_stats(db: *mut tidesdb_t, stats: *mut tidesdb_cache_stats_t) -> c_int; + pub fn tidesdb_stall_reason_name(reason: c_int) -> *const c_char; + pub fn tidesdb_get_stall_stats(db: *mut tidesdb_t, stats: *mut tidesdb_stall_stats_t) -> c_int; + pub fn tidesdb_io_class_name(cls: c_int) -> *const c_char; + pub fn tidesdb_get_io_stats(db: *mut tidesdb_t, stats: *mut tidesdb_io_stats_t) -> c_int; + pub fn tidesdb_range_stats( + db: *mut tidesdb_t, cf: *mut tidesdb_column_family_t, key_a: *const u8, key_a_size: size_t, key_b: *const u8, key_b_size: size_t, - cost: *mut c_double, - ) -> c_int; - - // Purge operations - pub fn tidesdb_purge_cf(cf: *mut tidesdb_column_family_t) -> c_int; - pub fn tidesdb_purge(db: *mut tidesdb_t) -> c_int; - - // WAL sync - pub fn tidesdb_sync_wal(cf: *mut tidesdb_column_family_t) -> c_int; - - // Database-level statistics - pub fn tidesdb_get_db_stats(db: *mut tidesdb_t, stats: *mut tidesdb_db_stats_t) -> c_int; - - // Iterator key+value combined - pub fn tidesdb_iter_key_value( - iter: *mut tidesdb_iter_t, - key: *mut *mut u8, - key_size: *mut size_t, - value: *mut *mut u8, - value_size: *mut size_t, + out: *mut tidesdb_range_stats_t, ) -> c_int; - - // Promote replica to primary - pub fn tidesdb_promote_to_primary(db: *mut tidesdb_t) -> c_int; - - // Object store connector factories - pub fn tidesdb_objstore_fs_create(root_dir: *const c_char) -> *mut tidesdb_objstore_t; - - // Object store default config - pub fn tidesdb_objstore_default_config() -> tidesdb_objstore_config_t; - - // Built-in comparator functions (raw symbols, usable as comparator_fn_cached) - pub fn tidesdb_comparator_memcmp( - key1: *const u8, - key1_size: size_t, - key2: *const u8, - key2_size: size_t, - ctx: *mut c_void, - ) -> c_int; - pub fn tidesdb_comparator_lexicographic( - key1: *const u8, - key1_size: size_t, - key2: *const u8, - key2_size: size_t, - ctx: *mut c_void, - ) -> c_int; - pub fn tidesdb_comparator_uint64( - key1: *const u8, - key1_size: size_t, - key2: *const u8, - key2_size: size_t, - ctx: *mut c_void, - ) -> c_int; - pub fn tidesdb_comparator_int64( - key1: *const u8, - key1_size: size_t, - key2: *const u8, - key2_size: size_t, - ctx: *mut c_void, - ) -> c_int; - pub fn tidesdb_comparator_reverse_memcmp( - key1: *const u8, - key1_size: size_t, - key2: *const u8, - key2_size: size_t, - ctx: *mut c_void, - ) -> c_int; - pub fn tidesdb_comparator_case_insensitive( - key1: *const u8, - key1_size: size_t, - key2: *const u8, - key2_size: size_t, - ctx: *mut c_void, - ) -> c_int; - - // Generic free - pub fn tidesdb_free(ptr: *mut c_void); } -// S3-compatible object store connector factories. -// -// These symbols are only present when the C library was built with -// `TIDESDB_WITH_S3=ON`, so they are gated behind the `objectstore` feature -// to avoid unresolved symbols at link time. -#[cfg(feature = "objectstore")] -#[link(name = "tidesdb")] -unsafe extern "C" { - pub fn tidesdb_objstore_s3_create( - endpoint: *const c_char, - bucket: *const c_char, - prefix: *const c_char, - access_key: *const c_char, - secret_key: *const c_char, - region: *const c_char, - use_ssl: c_int, - use_path_style: c_int, - ) -> *mut tidesdb_objstore_t; - - pub fn tidesdb_objstore_s3_create_config( - config: *const tidesdb_objstore_s3_config_t, - ) -> *mut tidesdb_objstore_t; +#[cfg(test)] +mod layout_tests { + use super::*; + use std::mem::{offset_of, size_of}; + + // Sizes and tail offsets for LP64 targets (Linux/macOS x86_64 and aarch64), taken from a C + // program compiled against db.h. A mismatch means the C header changed and the struct above + // must be brought back in step. + #[cfg(all(target_pointer_width = "64", not(windows)))] + #[test] + fn struct_layouts_match_c() { + assert_eq!(size_of::(), 224); + assert_eq!( + offset_of!(tidesdb_column_family_config_t, commit_hook_ctx), + 216 + ); + assert_eq!(size_of::(), 120); + assert_eq!(offset_of!(tidesdb_config_t, txn_timeout_seconds), 112); + assert_eq!(size_of::(), 48); + assert_eq!(size_of::(), 56); + assert_eq!(size_of::(), 40); + assert_eq!(size_of::(), 120); + assert_eq!(size_of::(), 96); + assert_eq!(size_of::(), 24); + assert_eq!(size_of::(), 24); + assert_eq!(size_of::(), 616); + assert_eq!(offset_of!(tidesdb_cf_stats_t, level_sizes), 232); + assert_eq!(offset_of!(tidesdb_cf_stats_t, filter_resident_bytes), 608); + assert_eq!(size_of::(), 288); + assert_eq!( + offset_of!(tidesdb_db_stats_t, write_stall_ceiling_hits), + 280 + ); + } } diff --git a/src/iterator.rs b/src/iterator.rs index 63840d8..52cd9de 100644 --- a/src/iterator.rs +++ b/src/iterator.rs @@ -5,14 +5,21 @@ //! Iterator operations for TidesDB. -use crate::error::{check_result, Error, Result}; +use crate::db::{take_c_buffer, ttl_from_c}; +use crate::error::{Error, Result, check_result}; use crate::ffi; +use crate::transaction::Transaction; +use std::marker::PhantomData; use std::ptr; -/// An iterator for traversing key-value pairs in a column family. +/// A bidirectional cursor over a column family at a transaction's snapshot. /// -/// Provides efficient bidirectional traversal. -/// The iterator is automatically freed when dropped. +/// Created by [`Transaction::new_iterator`] or [`Transaction::new_range_iterator`] and +/// freed when dropped. It borrows the transaction, so it must be dropped before the +/// transaction commits. +/// +/// Positioning calls (`seek*`, `next`, `prev`) succeed when they run off either end; check +/// [`is_valid`](Self::is_valid) to see whether the iterator is on an entry. /// /// # Example /// @@ -23,154 +30,109 @@ use std::ptr; /// db.create_column_family("my_cf", ColumnFamilyConfig::default())?; /// let cf = db.get_column_family("my_cf")?; /// -/// let mut txn = db.begin_transaction()?; -/// -/// // Insert some data -/// txn.put(&cf, b"key1", b"value1", -1)?; -/// txn.put(&cf, b"key2", b"value2", -1)?; -/// txn.commit()?; -/// -/// // Iterate /// let txn = db.begin_transaction()?; /// let mut iter = txn.new_iterator(&cf)?; /// iter.seek_to_first()?; -/// /// while iter.is_valid() { -/// let key = iter.key()?; -/// let value = iter.value()?; -/// println!("Key: {:?}, Value: {:?}", key, value); +/// let (key, value) = iter.key_value()?; +/// println!("{:?} = {:?}", key, value); /// iter.next()?; /// } /// # Ok::<(), tidesdb::Error>(()) /// ``` -pub struct Iterator { +pub struct Iterator<'txn> { iter: *mut ffi::tidesdb_iter_t, + _marker: PhantomData<&'txn Transaction<'txn>>, } -// Iterator uses internal locking for thread safety -unsafe impl Send for Iterator {} +/// A positioning result of NOT_FOUND only means the iterator ran off the end. +fn check_position(result: i32, context: &'static str) -> Result<()> { + if result == ffi::TDB_SUCCESS || result == ffi::TDB_ERR_NOT_FOUND { + Ok(()) + } else { + Err(Error::from_code(result, context)) + } +} -impl Iterator { - /// Create a new iterator wrapper. - pub(crate) fn new(iter: *mut ffi::tidesdb_iter_t) -> Self { - Iterator { iter } +impl Iterator<'_> { + /// Wrap a C iterator. + /// + /// # Safety + /// + /// `iter` must be a live iterator owned by the caller. + pub(crate) unsafe fn from_raw(iter: *mut ffi::tidesdb_iter_t) -> Self { + Iterator { + iter, + _marker: PhantomData, + } } /// Positions the iterator at the first key. pub fn seek_to_first(&mut self) -> Result<()> { let result = unsafe { ffi::tidesdb_iter_seek_to_first(self.iter) }; - check_result(result, "failed to seek to first") + check_position(result, "failed to seek to first") } /// Positions the iterator at the last key. pub fn seek_to_last(&mut self) -> Result<()> { let result = unsafe { ffi::tidesdb_iter_seek_to_last(self.iter) }; - check_result(result, "failed to seek to last") + check_position(result, "failed to seek to last") } - /// Positions the iterator at the first key >= target key. - /// - /// # Arguments - /// - /// * `key` - The target key + /// Positions the iterator at the first key >= `key`. pub fn seek(&mut self, key: &[u8]) -> Result<()> { - let key_ptr = if key.is_empty() { - ptr::null() - } else { - key.as_ptr() - }; - - let result = unsafe { ffi::tidesdb_iter_seek(self.iter, key_ptr, key.len()) }; - check_result(result, "failed to seek") + let result = unsafe { ffi::tidesdb_iter_seek(self.iter, key.as_ptr(), key.len()) }; + check_position(result, "failed to seek") } - /// Positions the iterator at the last key <= target key. - /// - /// # Arguments - /// - /// * `key` - The target key + /// Positions the iterator at the last key <= `key`. pub fn seek_for_prev(&mut self, key: &[u8]) -> Result<()> { - let key_ptr = if key.is_empty() { - ptr::null() - } else { - key.as_ptr() - }; - - let result = unsafe { ffi::tidesdb_iter_seek_for_prev(self.iter, key_ptr, key.len()) }; - check_result(result, "failed to seek for prev") + let result = unsafe { ffi::tidesdb_iter_seek_for_prev(self.iter, key.as_ptr(), key.len()) }; + check_position(result, "failed to seek for prev") } - /// Returns true if the iterator is positioned at a valid entry. + /// Whether the iterator is positioned on an entry. pub fn is_valid(&self) -> bool { unsafe { ffi::tidesdb_iter_valid(self.iter) != 0 } } - /// Moves the iterator to the next entry. - /// - /// Returns Ok(()) even when reaching the end of iteration. - /// Use `is_valid()` to check if the iterator is still positioned at a valid entry. + /// Moves to the next entry. + #[allow(clippy::should_implement_trait)] pub fn next(&mut self) -> Result<()> { let result = unsafe { ffi::tidesdb_iter_next(self.iter) }; - // NOT_FOUND is expected when reaching end of iteration - if result == ffi::TDB_ERR_NOT_FOUND || result == ffi::TDB_SUCCESS { - Ok(()) - } else { - Err(crate::error::Error::from_code(result, "failed to move to next")) - } + check_position(result, "failed to move to next") } - /// Moves the iterator to the previous entry. - /// - /// Returns Ok(()) even when reaching the beginning of iteration. - /// Use `is_valid()` to check if the iterator is still positioned at a valid entry. + /// Moves to the previous entry. pub fn prev(&mut self) -> Result<()> { let result = unsafe { ffi::tidesdb_iter_prev(self.iter) }; - // NOT_FOUND is expected when reaching beginning of iteration - if result == ffi::TDB_ERR_NOT_FOUND || result == ffi::TDB_SUCCESS { - Ok(()) - } else { - Err(crate::error::Error::from_code(result, "failed to move to prev")) - } + check_position(result, "failed to move to prev") } - /// Retrieves the current key from the iterator. - /// - /// # Returns - /// - /// The current key. + /// The key at the current position. pub fn key(&self) -> Result> { let mut key_ptr: *mut u8 = ptr::null_mut(); let mut key_len: usize = 0; - let result = unsafe { ffi::tidesdb_iter_key(self.iter, &mut key_ptr, &mut key_len) }; check_result(result, "failed to get key")?; + Ok(unsafe { take_c_buffer(key_ptr, key_len) }) + } - if key_ptr.is_null() { - return Err(Error::NullPointer("key")); - } - - let key = unsafe { - let slice = std::slice::from_raw_parts(key_ptr, key_len); - slice.to_vec() - }; - - Ok(key) + /// The value at the current position. + pub fn value(&self) -> Result> { + let mut value_ptr: *mut u8 = ptr::null_mut(); + let mut value_len: usize = 0; + let result = unsafe { ffi::tidesdb_iter_value(self.iter, &mut value_ptr, &mut value_len) }; + check_result(result, "failed to get value")?; + Ok(unsafe { take_c_buffer(value_ptr, value_len) }) } - /// Retrieves the current key and value from the iterator in a single call. - /// - /// This is more efficient than calling `key()` and `value()` separately - /// as it requires only one FFI call. - /// - /// # Returns - /// - /// A tuple of (key, value). + /// The key and value at the current position, in one call. pub fn key_value(&self) -> Result<(Vec, Vec)> { let mut key_ptr: *mut u8 = ptr::null_mut(); let mut key_len: usize = 0; let mut value_ptr: *mut u8 = ptr::null_mut(); let mut value_len: usize = 0; - let result = unsafe { ffi::tidesdb_iter_key_value( self.iter, @@ -181,52 +143,23 @@ impl Iterator { ) }; check_result(result, "failed to get key-value")?; - - if key_ptr.is_null() { - return Err(Error::NullPointer("key")); - } - if value_ptr.is_null() { - return Err(Error::NullPointer("value")); - } - - let key = unsafe { std::slice::from_raw_parts(key_ptr, key_len) }.to_vec(); - let value = unsafe { std::slice::from_raw_parts(value_ptr, value_len) }.to_vec(); - + let key = unsafe { take_c_buffer(key_ptr, key_len) }; + let value = unsafe { take_c_buffer(value_ptr, value_len) }; Ok((key, value)) } - /// Retrieves the current value from the iterator. - /// - /// # Returns - /// - /// The current value. - pub fn value(&self) -> Result> { - let mut value_ptr: *mut u8 = ptr::null_mut(); - let mut value_len: usize = 0; - - let result = unsafe { ffi::tidesdb_iter_value(self.iter, &mut value_ptr, &mut value_len) }; - check_result(result, "failed to get value")?; - - if value_ptr.is_null() { - return Err(Error::NullPointer("value")); - } - - let value = unsafe { - let slice = std::slice::from_raw_parts(value_ptr, value_len); - slice.to_vec() - }; - - Ok(value) + /// The current entry's expiry as an absolute Unix timestamp in seconds, `None` if it + /// never expires. + pub fn expires_at(&self) -> Result> { + let mut ttl: libc::time_t = -1; + let result = unsafe { ffi::tidesdb_iter_ttl(self.iter, &mut ttl) }; + check_result(result, "failed to get ttl")?; + Ok(ttl_from_c(ttl)) } } -impl Drop for Iterator { +impl Drop for Iterator<'_> { fn drop(&mut self) { - if !self.iter.is_null() { - unsafe { - ffi::tidesdb_iter_free(self.iter); - } - self.iter = ptr::null_mut(); - } + unsafe { ffi::tidesdb_iter_free(self.iter) }; } } diff --git a/src/lib.rs b/src/lib.rs index ee78082..9691227 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -17,65 +17,56 @@ //! # TidesDB //! -//! Safe Rust bindings for TidesDB - A high-performance embedded key-value storage engine. +//! Safe Rust bindings for TidesDB, a high-performance embeddable key-value storage engine. //! -//! TidesDB is a fast and efficient key-value storage engine library written in C. -//! The underlying data structure is based on a log-structured merge-tree (LSM-tree). -//! This Rust binding provides a safe, idiomatic Rust interface to TidesDB with full -//! support for all features. +//! TidesDB is written in C and based on a log-structured merge-tree (LSM-tree). This crate +//! binds TidesDB 10 and bundles its source, so no system install is needed. Crate majors map +//! to TidesDB majors: crate N.x binds TidesDB (N+9).x (1.x ↔ 10.x, 2.x ↔ 11.x). //! //! ## Features //! //! - MVCC with five isolation levels from READ UNCOMMITTED to SERIALIZABLE -//! - Column families (isolated key-value stores with independent configuration) -//! - Bidirectional iterators with forward/backward traversal and seek support -//! - TTL (time to live) support with automatic key expiration -//! - LZ4, LZ4 Fast, ZSTD, Snappy, or no compression -//! - Bloom filters with configurable false positive rates -//! - Global block CLOCK cache for hot blocks -//! - Savepoints for partial transaction rollback -//! - Six built-in comparators plus custom registration +//! - Snapshots and point-in-time reads +//! - Two-phase commit (prepare / commit prepared / recovery) +//! - Column families sharing one memtable, write-ahead log and value log +//! - Range and prefix deletes that cost one entry however many keys they cover +//! - Bidirectional iterators, including range-bounded iterators for narrow scans +//! - TTL with automatic key expiration +//! - Optional LZ4, LZ4 Fast, Zstd and Snappy compression pipelines +//! - Bloom filters, a shared block cache, savepoints, commit hooks for change data capture +//! - Detailed statistics: per family, database, cache, stalls, I/O and encoding +//! +//! Keys are always ordered byte-wise. To sort by another order, encode keys so their bytes +//! compare in that order (for example big-endian integers). //! //! ## Example //! //! ```no_run -//! use tidesdb::{TidesDB, Config, ColumnFamilyConfig, IsolationLevel}; +//! use tidesdb::{TidesDB, Config, ColumnFamilyConfig}; +//! use std::time::Duration; //! //! fn main() -> tidesdb::Result<()> { -//! // Open database -//! let config = Config::new("./mydb") -//! .num_flush_threads(2) -//! .num_compaction_threads(2); -//! -//! let db = TidesDB::open(config)?; -//! -//! // Create a column family -//! let cf_config = ColumnFamilyConfig::default(); -//! db.create_column_family("my_cf", cf_config)?; -//! -//! // Get the column family +//! let db = TidesDB::open(Config::new("./mydb"))?; +//! db.create_column_family("my_cf", ColumnFamilyConfig::default())?; //! let cf = db.get_column_family("my_cf")?; //! //! // Write data in a transaction //! let mut txn = db.begin_transaction()?; -//! txn.put(&cf, b"key1", b"value1", -1)?; -//! txn.put(&cf, b"key2", b"value2", -1)?; +//! txn.put(&cf, b"key1", b"value1")?; +//! txn.put_with_ttl(&cf, b"session", b"abc", Duration::from_secs(60))?; //! txn.commit()?; //! -//! // Read data +//! // Read it back //! let txn = db.begin_transaction()?; //! let value = txn.get(&cf, b"key1")?; //! println!("Value: {:?}", String::from_utf8_lossy(&value)); //! -//! // Iterate over data +//! // Iterate //! let mut iter = txn.new_iterator(&cf)?; //! iter.seek_to_first()?; //! while iter.is_valid() { -//! let key = iter.key()?; -//! let value = iter.value()?; -//! println!("Key: {:?}, Value: {:?}", -//! String::from_utf8_lossy(&key), -//! String::from_utf8_lossy(&value)); +//! let (key, value) = iter.key_value()?; +//! println!("{:?} = {:?}", key, value); //! iter.next()?; //! } //! @@ -91,2002 +82,29 @@ mod iterator; mod stats; mod transaction; -// Re-export public types +#[cfg(test)] +mod tests; + pub use config::{ - ColumnFamilyConfig, CompressionAlgorithm, Config, IsolationLevel, LogLevel, ObjectStoreConfig, - SyncMode, + ColumnFamilyConfig, CompressionAlgorithm, Config, IsolationLevel, LogLevel, SyncMode, +}; +pub use db::{ + ColumnFamily, CommitOp, PreparedTransaction, Snapshot, TidesDB, raise_open_file_limit, }; -#[cfg(feature = "objectstore")] -pub use config::S3Config; -#[cfg(tidesdb_has_raise_open_file_limit)] -pub use db::raise_open_file_limit; -pub use db::{ColumnFamily, CommitOp, TidesDB, finalize, free, init, init_with_allocator}; pub use error::{Error, ErrorCode, Result}; -pub use ffi::{tidesdb_calloc_fn, tidesdb_free_fn, tidesdb_malloc_fn, tidesdb_realloc_fn}; pub use iterator::Iterator; -pub use stats::{CacheStats, DbStats, Stats}; -pub use transaction::Transaction; - -#[cfg(test)] -mod tests { - use super::*; - use std::time::{SystemTime, UNIX_EPOCH}; - use tempfile::TempDir; - - fn create_test_db() -> (TidesDB, TempDir) { - let temp_dir = TempDir::new().unwrap(); - let config = Config::new(temp_dir.path()) - .num_flush_threads(2) - .num_compaction_threads(2) - .log_level(LogLevel::Info) - .block_cache_size(64 * 1024 * 1024) - .max_open_sstables(256); - - let db = TidesDB::open(config).unwrap(); - (db, temp_dir) - } - - #[test] - fn test_open_close() { - let (db, _temp_dir) = create_test_db(); - drop(db); - } - - #[test] - fn test_create_drop_column_family() { - let (db, _temp_dir) = create_test_db(); - - let cf_config = ColumnFamilyConfig::default(); - db.create_column_family("test_cf", cf_config).unwrap(); - - let cf = db.get_column_family("test_cf").unwrap(); - assert_eq!(cf.name(), "test_cf"); - - let families = db.list_column_families().unwrap(); - assert!(families.contains(&"test_cf".to_string())); - - db.drop_column_family("test_cf").unwrap(); - } - - #[test] - fn test_transaction_put_get_delete() { - let (db, _temp_dir) = create_test_db(); - - let cf_config = ColumnFamilyConfig::default(); - db.create_column_family("test_cf", cf_config).unwrap(); - let cf = db.get_column_family("test_cf").unwrap(); - - // Put - { - let mut txn = db.begin_transaction().unwrap(); - txn.put(&cf, b"key", b"value", -1).unwrap(); - txn.commit().unwrap(); - } - - // Get - { - let txn = db.begin_transaction().unwrap(); - let value = txn.get(&cf, b"key").unwrap(); - assert_eq!(value, b"value"); - } - - // Delete - { - let mut txn = db.begin_transaction().unwrap(); - txn.delete(&cf, b"key").unwrap(); - txn.commit().unwrap(); - } - - // Verify deleted - { - let txn = db.begin_transaction().unwrap(); - let result = txn.get(&cf, b"key"); - assert!(result.is_err()); - } - } - - #[test] - fn test_transaction_with_ttl() { - let (db, _temp_dir) = create_test_db(); - - let cf_config = ColumnFamilyConfig::default(); - db.create_column_family("test_cf", cf_config).unwrap(); - let cf = db.get_column_family("test_cf").unwrap(); - - // Set TTL to 2 seconds from now - let ttl = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_secs() as i64 - + 2; - - { - let mut txn = db.begin_transaction().unwrap(); - txn.put(&cf, b"temp_key", b"temp_value", ttl).unwrap(); - txn.commit().unwrap(); - } - - // Verify key exists before expiration - { - let txn = db.begin_transaction().unwrap(); - let value = txn.get(&cf, b"temp_key").unwrap(); - assert_eq!(value, b"temp_value"); - } - } - - #[test] - fn test_multi_operation_transaction() { - let (db, _temp_dir) = create_test_db(); - - let cf_config = ColumnFamilyConfig::default(); - db.create_column_family("test_cf", cf_config).unwrap(); - let cf = db.get_column_family("test_cf").unwrap(); - - // Multiple operations in one transaction - { - let mut txn = db.begin_transaction().unwrap(); - txn.put(&cf, b"key1", b"value1", -1).unwrap(); - txn.put(&cf, b"key2", b"value2", -1).unwrap(); - txn.put(&cf, b"key3", b"value3", -1).unwrap(); - txn.commit().unwrap(); - } - - // Verify all keys exist - { - let txn = db.begin_transaction().unwrap(); - for i in 1..=3 { - let key = format!("key{}", i); - let expected_value = format!("value{}", i); - let value = txn.get(&cf, key.as_bytes()).unwrap(); - assert_eq!(value, expected_value.as_bytes()); - } - } - } - - #[test] - fn test_transaction_rollback() { - let (db, _temp_dir) = create_test_db(); - - let cf_config = ColumnFamilyConfig::default(); - db.create_column_family("test_cf", cf_config).unwrap(); - let cf = db.get_column_family("test_cf").unwrap(); - - { - let mut txn = db.begin_transaction().unwrap(); - txn.put(&cf, b"rollback_key", b"rollback_value", -1) - .unwrap(); - txn.rollback().unwrap(); - } - - // Verify key does not exist - { - let txn = db.begin_transaction().unwrap(); - let result = txn.get(&cf, b"rollback_key"); - assert!(result.is_err()); - } - } - - #[test] - fn test_savepoints() { - let (db, _temp_dir) = create_test_db(); - - let cf_config = ColumnFamilyConfig::default(); - db.create_column_family("test_cf", cf_config).unwrap(); - let cf = db.get_column_family("test_cf").unwrap(); - - { - let mut txn = db.begin_transaction().unwrap(); - txn.put(&cf, b"key1", b"value1", -1).unwrap(); - - txn.savepoint("sp1").unwrap(); - txn.put(&cf, b"key2", b"value2", -1).unwrap(); - - // Rollback to savepoint -- key2 is discarded, key1 remains - txn.rollback_to_savepoint("sp1").unwrap(); - - // Add different operation after rollback - txn.put(&cf, b"key3", b"value3", -1).unwrap(); - - txn.commit().unwrap(); - } - - // Verify results - { - let txn = db.begin_transaction().unwrap(); - - // key1 should exist - assert!(txn.get(&cf, b"key1").is_ok()); - - // key2 should not exist (rolled back) - assert!(txn.get(&cf, b"key2").is_err()); - - // key3 should exist - assert!(txn.get(&cf, b"key3").is_ok()); - } - } - - #[test] - fn test_iterator() { - let (db, _temp_dir) = create_test_db(); - - let cf_config = ColumnFamilyConfig::default(); - db.create_column_family("test_cf", cf_config).unwrap(); - let cf = db.get_column_family("test_cf").unwrap(); - - // Insert some data - { - let mut txn = db.begin_transaction().unwrap(); - for i in 0..10 { - let key = format!("key{:02}", i); - let value = format!("value{}", i); - txn.put(&cf, key.as_bytes(), value.as_bytes(), -1).unwrap(); - } - txn.commit().unwrap(); - } - - // Forward iteration - { - let txn = db.begin_transaction().unwrap(); - let mut iter = txn.new_iterator(&cf).unwrap(); - iter.seek_to_first().unwrap(); - - let mut count = 0; - while iter.is_valid() { - let key = iter.key().unwrap(); - let value = iter.value().unwrap(); - assert!(!key.is_empty()); - assert!(!value.is_empty()); - count += 1; - iter.next().unwrap(); - } - assert_eq!(count, 10); - // iter and txn dropped here in correct order - } - - // Backward iteration in separate scope - { - let txn = db.begin_transaction().unwrap(); - let mut iter = txn.new_iterator(&cf).unwrap(); - iter.seek_to_last().unwrap(); - - let mut count = 0; - while iter.is_valid() { - let key = iter.key().unwrap(); - let value = iter.value().unwrap(); - assert!(!key.is_empty()); - assert!(!value.is_empty()); - count += 1; - iter.prev().unwrap(); - } - assert_eq!(count, 10); - } - } - - #[test] - fn test_isolation_levels() { - let (db, _temp_dir) = create_test_db(); - - let cf_config = ColumnFamilyConfig::default(); - db.create_column_family("test_cf", cf_config).unwrap(); - - // Test different isolation levels - for level in [ - IsolationLevel::ReadUncommitted, - IsolationLevel::ReadCommitted, - IsolationLevel::RepeatableRead, - IsolationLevel::Snapshot, - IsolationLevel::Serializable, - ] { - let txn = db.begin_transaction_with_isolation(level).unwrap(); - drop(txn); - } - } - - #[test] - fn test_column_family_stats() { - let (db, _temp_dir) = create_test_db(); - - let cf_config = ColumnFamilyConfig::default(); - db.create_column_family("test_cf", cf_config).unwrap(); - let cf = db.get_column_family("test_cf").unwrap(); - - // Insert some data - { - let mut txn = db.begin_transaction().unwrap(); - for i in 0..100 { - let key = format!("key{}", i); - let value = format!("value{}", i); - txn.put(&cf, key.as_bytes(), value.as_bytes(), -1).unwrap(); - } - txn.commit().unwrap(); - } - - let stats = cf.get_stats().unwrap(); - assert!(stats.num_levels >= 0); - } - - #[test] - fn test_cache_stats() { - let (db, _temp_dir) = create_test_db(); - - let stats = db.get_cache_stats().unwrap(); - // Just verify we can get stats without error - let _ = stats.enabled; - } - - #[test] - fn test_custom_column_family_config() { - let (db, _temp_dir) = create_test_db(); - - let cf_config = ColumnFamilyConfig::new() - .write_buffer_size(128 * 1024 * 1024) - .level_size_ratio(10) - .min_levels(5) - .compression_algorithm(CompressionAlgorithm::None) - .enable_bloom_filter(true) - .bloom_fpr(0.01) - .enable_block_indexes(true) - .sync_mode(SyncMode::Interval) - .sync_interval_us(128000) - .default_isolation_level(IsolationLevel::ReadCommitted); - - db.create_column_family("custom_cf", cf_config).unwrap(); - - let cf = db.get_column_family("custom_cf").unwrap(); - assert_eq!(cf.name(), "custom_cf"); - } - - #[test] - fn test_rename_column_family() { - let (db, _temp_dir) = create_test_db(); - - let cf_config = ColumnFamilyConfig::default(); - db.create_column_family("old_name", cf_config).unwrap(); - - // Insert some data - { - let cf = db.get_column_family("old_name").unwrap(); - let mut txn = db.begin_transaction().unwrap(); - txn.put(&cf, b"key", b"value", -1).unwrap(); - txn.commit().unwrap(); - } - - // Rename the column family - db.rename_column_family("old_name", "new_name").unwrap(); - - // Verify old name no longer exists - assert!(db.get_column_family("old_name").is_err()); - - // Verify new name exists and data is preserved - let cf = db.get_column_family("new_name").unwrap(); - let txn = db.begin_transaction().unwrap(); - let value = txn.get(&cf, b"key").unwrap(); - assert_eq!(value, b"value"); - } - - #[test] - fn test_backup() { - let (db, temp_dir) = create_test_db(); - - let cf_config = ColumnFamilyConfig::default(); - db.create_column_family("test_cf", cf_config).unwrap(); - - // Insert some data - { - let cf = db.get_column_family("test_cf").unwrap(); - let mut txn = db.begin_transaction().unwrap(); - txn.put(&cf, b"key", b"value", -1).unwrap(); - txn.commit().unwrap(); - } - - // Create backup directory - let backup_dir = temp_dir.path().join("backup"); - db.backup(backup_dir.to_str().unwrap()).unwrap(); - - // Verify backup directory exists - assert!(backup_dir.exists()); - } - - #[test] - fn test_checkpoint() { - let (db, temp_dir) = create_test_db(); - - let cf_config = ColumnFamilyConfig::default(); - db.create_column_family("test_cf", cf_config).unwrap(); - - // Insert some data - { - let cf = db.get_column_family("test_cf").unwrap(); - let mut txn = db.begin_transaction().unwrap(); - txn.put(&cf, b"key1", b"value1", -1).unwrap(); - txn.put(&cf, b"key2", b"value2", -1).unwrap(); - txn.commit().unwrap(); - } - - // Create checkpoint directory - let checkpoint_dir = temp_dir.path().join("checkpoint"); - db.checkpoint(checkpoint_dir.to_str().unwrap()).unwrap(); - - // Verify checkpoint directory exists - assert!(checkpoint_dir.exists()); - - // Open the checkpoint as a separate database and verify data - let checkpoint_config = Config::new(checkpoint_dir.to_str().unwrap()) - .num_flush_threads(2) - .num_compaction_threads(2) - .log_level(LogLevel::Info) - .block_cache_size(64 * 1024 * 1024) - .max_open_sstables(256); - - let checkpoint_db = TidesDB::open(checkpoint_config).unwrap(); - let cf = checkpoint_db.get_column_family("test_cf").unwrap(); - let txn = checkpoint_db.begin_transaction().unwrap(); - let v1 = txn.get(&cf, b"key1").unwrap(); - assert_eq!(v1, b"value1"); - let v2 = txn.get(&cf, b"key2").unwrap(); - assert_eq!(v2, b"value2"); - } - - #[test] - fn test_is_flushing_compacting() { - let (db, _temp_dir) = create_test_db(); - - let cf_config = ColumnFamilyConfig::default(); - db.create_column_family("test_cf", cf_config).unwrap(); - let cf = db.get_column_family("test_cf").unwrap(); - - // These should return false when no operations are in progress - // (just testing that the methods work without error) - let _ = cf.is_flushing(); - let _ = cf.is_compacting(); - } - - #[test] - fn test_update_runtime_config() { - let (db, _temp_dir) = create_test_db(); - - let cf_config = ColumnFamilyConfig::default(); - db.create_column_family("test_cf", cf_config).unwrap(); - let cf = db.get_column_family("test_cf").unwrap(); - - // Create new configuration - let new_config = ColumnFamilyConfig::new() - .write_buffer_size(256 * 1024 * 1024) - .compression_algorithm(CompressionAlgorithm::None); - - // Update runtime config - cf.update_runtime_config(&new_config, false).unwrap(); - } - - #[test] - fn test_extended_stats() { - let (db, _temp_dir) = create_test_db(); - - let cf_config = ColumnFamilyConfig::default(); - db.create_column_family("test_cf", cf_config).unwrap(); - let cf = db.get_column_family("test_cf").unwrap(); - - // Insert some data - { - let mut txn = db.begin_transaction().unwrap(); - for i in 0..100 { - let key = format!("key{}", i); - let value = format!("value{}", i); - txn.put(&cf, key.as_bytes(), value.as_bytes(), -1).unwrap(); - } - txn.commit().unwrap(); - } - - let stats = cf.get_stats().unwrap(); - - // Verify extended stats fields exist and are accessible - assert!(stats.num_levels >= 0); - let _ = stats.total_keys; - let _ = stats.total_data_size; - let _ = stats.avg_key_size; - let _ = stats.avg_value_size; - let _ = stats.read_amp; - let _ = stats.hit_rate; - let _ = stats.level_key_counts; - } - - #[test] - fn test_column_family_config_builders() { - // Test all the new builder methods - let config = ColumnFamilyConfig::new() - .write_buffer_size(64 * 1024 * 1024) - .level_size_ratio(10) - .min_levels(4) - .dividing_level_offset(2) - .klog_value_threshold(1024) - .compression_algorithm(CompressionAlgorithm::Lz4) - .enable_bloom_filter(true) - .bloom_fpr(0.01) - .enable_block_indexes(true) - .index_sample_ratio(16) - .block_index_prefix_len(4) - .sync_mode(SyncMode::Interval) - .sync_interval_us(128000) - .comparator_name("memcmp") - .skip_list_max_level(12) - .skip_list_probability(0.25) - .default_isolation_level(IsolationLevel::ReadCommitted) - .min_disk_space(1024 * 1024 * 1024) - .l1_file_count_trigger(4) - .l0_queue_stall_threshold(8) - .use_btree(false); - - // Verify some values - assert_eq!(config.write_buffer_size, 64 * 1024 * 1024); - assert_eq!(config.min_levels, 4); - assert_eq!(config.dividing_level_offset, 2); - assert_eq!(config.klog_value_threshold, 1024); - assert_eq!(config.compression_algorithm, CompressionAlgorithm::Lz4); - assert_eq!(config.index_sample_ratio, 16); - assert_eq!(config.block_index_prefix_len, 4); - assert_eq!(config.comparator_name, "memcmp"); - assert_eq!(config.skip_list_max_level, 12); - assert_eq!(config.min_disk_space, 1024 * 1024 * 1024); - assert_eq!(config.l1_file_count_trigger, 4); - assert_eq!(config.l0_queue_stall_threshold, 8); - assert_eq!(config.use_btree, false); - } - - #[test] - fn test_use_btree_config() { - let (db, _temp_dir) = create_test_db(); - - // Create column family with B+tree format enabled - let cf_config = ColumnFamilyConfig::new() - .use_btree(true) - .compression_algorithm(CompressionAlgorithm::None); - - db.create_column_family("btree_cf", cf_config).unwrap(); - - let cf = db.get_column_family("btree_cf").unwrap(); - assert_eq!(cf.name(), "btree_cf"); - - // Insert some data - { - let mut txn = db.begin_transaction().unwrap(); - for i in 0..50 { - let key = format!("key{:04}", i); - let value = format!("value{}", i); - txn.put(&cf, key.as_bytes(), value.as_bytes(), -1).unwrap(); - } - txn.commit().unwrap(); - } - - // Verify data can be read back - { - let txn = db.begin_transaction().unwrap(); - let value = txn.get(&cf, b"key0000").unwrap(); - assert_eq!(value, b"value0"); - } - } - - #[test] - fn test_btree_stats() { - let (db, _temp_dir) = create_test_db(); - - // Create column family with B+tree format - let cf_config = ColumnFamilyConfig::new().use_btree(true); - - db.create_column_family("btree_stats_cf", cf_config) - .unwrap(); - let cf = db.get_column_family("btree_stats_cf").unwrap(); - - // Insert some data - { - let mut txn = db.begin_transaction().unwrap(); - for i in 0..100 { - let key = format!("key{:04}", i); - let value = format!("value{}", i); - txn.put(&cf, key.as_bytes(), value.as_bytes(), -1).unwrap(); - } - txn.commit().unwrap(); - } - - // Get stats and verify B+tree fields are accessible - let stats = cf.get_stats().unwrap(); - - // Verify B+tree stats fields exist and are accessible - let _ = stats.use_btree; - let _ = stats.btree_total_nodes; - let _ = stats.btree_max_height; - let _ = stats.btree_avg_height; - - // Basic sanity check - assert!(stats.num_levels >= 0); - } - - #[test] - fn test_clone_column_family() { - let (db, _temp_dir) = create_test_db(); - - let cf_config = ColumnFamilyConfig::default(); - db.create_column_family("source_cf", cf_config).unwrap(); - - // Insert data into source - { - let cf = db.get_column_family("source_cf").unwrap(); - let mut txn = db.begin_transaction().unwrap(); - txn.put(&cf, b"key1", b"value1", -1).unwrap(); - txn.put(&cf, b"key2", b"value2", -1).unwrap(); - txn.put(&cf, b"key3", b"value3", -1).unwrap(); - txn.commit().unwrap(); - } - - // Clone the column family - db.clone_column_family("source_cf", "cloned_cf").unwrap(); - - // Verify cloned column family exists - let cloned_cf = db.get_column_family("cloned_cf").unwrap(); - assert_eq!(cloned_cf.name(), "cloned_cf"); - - // Verify data is present in the clone - { - let txn = db.begin_transaction().unwrap(); - let v1 = txn.get(&cloned_cf, b"key1").unwrap(); - assert_eq!(v1, b"value1"); - let v2 = txn.get(&cloned_cf, b"key2").unwrap(); - assert_eq!(v2, b"value2"); - let v3 = txn.get(&cloned_cf, b"key3").unwrap(); - assert_eq!(v3, b"value3"); - } - - // Verify source still exists and is independent - let source_cf = db.get_column_family("source_cf").unwrap(); - { - let txn = db.begin_transaction().unwrap(); - let v1 = txn.get(&source_cf, b"key1").unwrap(); - assert_eq!(v1, b"value1"); - } - - // Verify clone is independent -- write to clone, source unaffected - { - let mut txn = db.begin_transaction().unwrap(); - txn.put(&cloned_cf, b"key4", b"value4", -1).unwrap(); - txn.commit().unwrap(); - } - - { - let txn = db.begin_transaction().unwrap(); - // key4 should exist in clone - let v4 = txn.get(&cloned_cf, b"key4").unwrap(); - assert_eq!(v4, b"value4"); - // key4 should not exist in source - assert!(txn.get(&source_cf, b"key4").is_err()); - } - } - - #[test] - fn test_clone_column_family_errors() { - let (db, _temp_dir) = create_test_db(); - - let cf_config = ColumnFamilyConfig::default(); - db.create_column_family("source_cf", cf_config).unwrap(); - - // Clone to a name that already exists should fail - let cf_config2 = ColumnFamilyConfig::default(); - db.create_column_family("existing_cf", cf_config2).unwrap(); - assert!(db.clone_column_family("source_cf", "existing_cf").is_err()); - - // Clone from non-existent source should fail - assert!(db.clone_column_family("nonexistent_cf", "new_cf").is_err()); - } - - #[test] - fn test_transaction_reset() { - let (db, _temp_dir) = create_test_db(); - - let cf_config = ColumnFamilyConfig::default(); - db.create_column_family("test_cf", cf_config).unwrap(); - let cf = db.get_column_family("test_cf").unwrap(); - - // Begin transaction, do work, commit, then reset and reuse - let mut txn = db.begin_transaction().unwrap(); - - // First batch - txn.put(&cf, b"key1", b"value1", -1).unwrap(); - txn.commit().unwrap(); - - // Reset with same isolation level - txn.reset(IsolationLevel::ReadCommitted).unwrap(); - - // Second batch using the same transaction - txn.put(&cf, b"key2", b"value2", -1).unwrap(); - txn.commit().unwrap(); - - // Verify both keys exist - { - let txn = db.begin_transaction().unwrap(); - let v1 = txn.get(&cf, b"key1").unwrap(); - assert_eq!(v1, b"value1"); - let v2 = txn.get(&cf, b"key2").unwrap(); - assert_eq!(v2, b"value2"); - } - } - - #[test] - fn test_transaction_reset_with_different_isolation() { - let (db, _temp_dir) = create_test_db(); - - let cf_config = ColumnFamilyConfig::default(); - db.create_column_family("test_cf", cf_config).unwrap(); - let cf = db.get_column_family("test_cf").unwrap(); - - // Begin with ReadCommitted - let mut txn = db - .begin_transaction_with_isolation(IsolationLevel::ReadCommitted) - .unwrap(); - txn.put(&cf, b"key1", b"value1", -1).unwrap(); - txn.commit().unwrap(); - - // Reset with different isolation level - txn.reset(IsolationLevel::RepeatableRead).unwrap(); - txn.put(&cf, b"key2", b"value2", -1).unwrap(); - txn.commit().unwrap(); - - // Reset again with yet another level - txn.reset(IsolationLevel::Snapshot).unwrap(); - txn.put(&cf, b"key3", b"value3", -1).unwrap(); - txn.commit().unwrap(); - - // Verify all keys - { - let txn = db.begin_transaction().unwrap(); - assert_eq!(txn.get(&cf, b"key1").unwrap(), b"value1"); - assert_eq!(txn.get(&cf, b"key2").unwrap(), b"value2"); - assert_eq!(txn.get(&cf, b"key3").unwrap(), b"value3"); - } - } - - #[test] - fn test_transaction_reset_after_rollback() { - let (db, _temp_dir) = create_test_db(); - - let cf_config = ColumnFamilyConfig::default(); - db.create_column_family("test_cf", cf_config).unwrap(); - let cf = db.get_column_family("test_cf").unwrap(); - - let mut txn = db.begin_transaction().unwrap(); - - // Do work and rollback - txn.put(&cf, b"key1", b"value1", -1).unwrap(); - txn.rollback().unwrap(); - - // Reset after rollback should work - txn.reset(IsolationLevel::ReadCommitted).unwrap(); - - // Do new work and commit - txn.put(&cf, b"key2", b"value2", -1).unwrap(); - txn.commit().unwrap(); - - // Verify key1 does not exist (was rolled back), key2 exists - { - let txn = db.begin_transaction().unwrap(); - assert!(txn.get(&cf, b"key1").is_err()); - assert_eq!(txn.get(&cf, b"key2").unwrap(), b"value2"); - } - } - - #[test] - fn test_range_cost() { - let (db, _temp_dir) = create_test_db(); - - let cf_config = ColumnFamilyConfig::default(); - db.create_column_family("test_cf", cf_config).unwrap(); - let cf = db.get_column_family("test_cf").unwrap(); - - // Insert some data - { - let mut txn = db.begin_transaction().unwrap(); - for i in 0..100 { - let key = format!("key{:04}", i); - let value = format!("value{}", i); - txn.put(&cf, key.as_bytes(), value.as_bytes(), -1).unwrap(); - } - txn.commit().unwrap(); - } - - // Estimate range cost - let cost = cf.range_cost(b"key0000", b"key0099").unwrap(); - // Cost should be non-negative - assert!(cost >= 0.0); - } - - #[test] - fn test_range_cost_comparison() { - let (db, _temp_dir) = create_test_db(); - - let cf_config = ColumnFamilyConfig::default(); - db.create_column_family("test_cf", cf_config).unwrap(); - let cf = db.get_column_family("test_cf").unwrap(); - - // Insert data - { - let mut txn = db.begin_transaction().unwrap(); - for i in 0..1000 { - let key = format!("key{:06}", i); - let value = format!("value{}", i); - txn.put(&cf, key.as_bytes(), value.as_bytes(), -1).unwrap(); - } - txn.commit().unwrap(); - } - - // Compare costs of different ranges - let cost_small = cf.range_cost(b"key000000", b"key000009").unwrap(); - let cost_large = cf.range_cost(b"key000000", b"key000999").unwrap(); - - // Both should be non-negative - assert!(cost_small >= 0.0); - assert!(cost_large >= 0.0); - } - - #[test] - fn test_range_cost_key_order_invariant() { - let (db, _temp_dir) = create_test_db(); - - let cf_config = ColumnFamilyConfig::default(); - db.create_column_family("test_cf", cf_config).unwrap(); - let cf = db.get_column_family("test_cf").unwrap(); - - // Insert some data - { - let mut txn = db.begin_transaction().unwrap(); - for i in 0..50 { - let key = format!("key{:04}", i); - let value = format!("value{}", i); - txn.put(&cf, key.as_bytes(), value.as_bytes(), -1).unwrap(); - } - txn.commit().unwrap(); - } - - // Key order should not matter -- both orderings produce the same cost - let cost_ab = cf.range_cost(b"key0000", b"key0049").unwrap(); - let cost_ba = cf.range_cost(b"key0049", b"key0000").unwrap(); - assert!((cost_ab - cost_ba).abs() < f64::EPSILON); - } - - #[test] - fn test_range_cost_empty_range() { - let (db, _temp_dir) = create_test_db(); - - let cf_config = ColumnFamilyConfig::default(); - db.create_column_family("test_cf", cf_config).unwrap(); - let cf = db.get_column_family("test_cf").unwrap(); - - // Cost on empty column family should be 0.0 - let cost = cf.range_cost(b"key_a", b"key_b").unwrap(); - assert!(cost >= 0.0); - } - - #[test] - fn test_commit_hook_basic() { - use std::sync::{Arc, Mutex}; - - let (db, _temp_dir) = create_test_db(); - - let cf_config = ColumnFamilyConfig::default(); - db.create_column_family("test_cf", cf_config).unwrap(); - let mut cf = db.get_column_family("test_cf").unwrap(); - - // Track operations received by the hook - let ops_log: Arc, Option>, bool)>>> = - Arc::new(Mutex::new(Vec::new())); - let ops_log_clone = ops_log.clone(); - - // Set commit hook - cf.set_commit_hook(move |ops, _commit_seq| { - let mut log = ops_log_clone.lock().unwrap(); - for op in ops { - log.push((op.key.clone(), op.value.clone(), op.is_delete)); - } - 0 - }) - .unwrap(); - - // Commit a transaction -- hook should fire - { - let mut txn = db.begin_transaction().unwrap(); - txn.put(&cf, b"key1", b"value1", -1).unwrap(); - txn.commit().unwrap(); - } - - // Verify hook received the operation - let log = ops_log.lock().unwrap(); - assert!(!log.is_empty()); - assert_eq!(log[0].0, b"key1"); - assert_eq!(log[0].1, Some(b"value1".to_vec())); - assert!(!log[0].2); // not a delete - } - - #[test] - fn test_commit_hook_delete_operations() { - use std::sync::{Arc, Mutex}; - - let (db, _temp_dir) = create_test_db(); - - let cf_config = ColumnFamilyConfig::default(); - db.create_column_family("test_cf", cf_config).unwrap(); - let mut cf = db.get_column_family("test_cf").unwrap(); - - // Insert data first - { - let mut txn = db.begin_transaction().unwrap(); - txn.put(&cf, b"key1", b"value1", -1).unwrap(); - txn.commit().unwrap(); - } - - // Now set hook and delete - let ops_log: Arc, bool)>>> = Arc::new(Mutex::new(Vec::new())); - let ops_log_clone = ops_log.clone(); - - cf.set_commit_hook(move |ops, _commit_seq| { - let mut log = ops_log_clone.lock().unwrap(); - for op in ops { - log.push((op.key.clone(), op.is_delete)); - } - 0 - }) - .unwrap(); - - { - let mut txn = db.begin_transaction().unwrap(); - txn.delete(&cf, b"key1").unwrap(); - txn.commit().unwrap(); - } - - let log = ops_log.lock().unwrap(); - assert!(!log.is_empty()); - // Find the delete operation - let delete_op = log.iter().find(|(k, _)| k == b"key1"); - assert!(delete_op.is_some()); - assert!(delete_op.unwrap().1); // is_delete should be true - } - - #[test] - fn test_commit_hook_sequence_numbers() { - use std::sync::{Arc, Mutex}; - - let (db, _temp_dir) = create_test_db(); - - let cf_config = ColumnFamilyConfig::default(); - db.create_column_family("test_cf", cf_config).unwrap(); - let mut cf = db.get_column_family("test_cf").unwrap(); - - let seqs: Arc>> = Arc::new(Mutex::new(Vec::new())); - let seqs_clone = seqs.clone(); - - cf.set_commit_hook(move |_ops, commit_seq| { - let mut s = seqs_clone.lock().unwrap(); - s.push(commit_seq); - 0 - }) - .unwrap(); - - // Multiple commits - for i in 0..3 { - let mut txn = db.begin_transaction().unwrap(); - let key = format!("key{}", i); - let value = format!("value{}", i); - txn.put(&cf, key.as_bytes(), value.as_bytes(), -1).unwrap(); - txn.commit().unwrap(); - } - - let seqs = seqs.lock().unwrap(); - assert_eq!(seqs.len(), 3); - // Sequence numbers should be monotonically increasing - for i in 1..seqs.len() { - assert!(seqs[i] > seqs[i - 1]); - } - } - - #[test] - fn test_commit_hook_clear() { - use std::sync::{Arc, Mutex}; - - let (db, _temp_dir) = create_test_db(); - - let cf_config = ColumnFamilyConfig::default(); - db.create_column_family("test_cf", cf_config).unwrap(); - let mut cf = db.get_column_family("test_cf").unwrap(); - - let count: Arc> = Arc::new(Mutex::new(0)); - let count_clone = count.clone(); - - cf.set_commit_hook(move |_ops, _commit_seq| { - let mut c = count_clone.lock().unwrap(); - *c += 1; - 0 - }) - .unwrap(); - - // First commit -- hook fires - { - let mut txn = db.begin_transaction().unwrap(); - txn.put(&cf, b"key1", b"value1", -1).unwrap(); - txn.commit().unwrap(); - } - assert_eq!(*count.lock().unwrap(), 1); - - // Clear the hook - cf.clear_commit_hook().unwrap(); - - // Second commit -- hook should NOT fire - { - let mut txn = db.begin_transaction().unwrap(); - txn.put(&cf, b"key2", b"value2", -1).unwrap(); - txn.commit().unwrap(); - } - assert_eq!(*count.lock().unwrap(), 1); // Still 1, not 2 - } - - #[test] - fn test_commit_hook_replace() { - use std::sync::{Arc, Mutex}; - - let (db, _temp_dir) = create_test_db(); - - let cf_config = ColumnFamilyConfig::default(); - db.create_column_family("test_cf", cf_config).unwrap(); - let mut cf = db.get_column_family("test_cf").unwrap(); - - let hook1_count: Arc> = Arc::new(Mutex::new(0)); - let hook2_count: Arc> = Arc::new(Mutex::new(0)); - - // Set first hook - let h1 = hook1_count.clone(); - cf.set_commit_hook(move |_ops, _commit_seq| { - *h1.lock().unwrap() += 1; - 0 - }) - .unwrap(); - - { - let mut txn = db.begin_transaction().unwrap(); - txn.put(&cf, b"key1", b"value1", -1).unwrap(); - txn.commit().unwrap(); - } - assert_eq!(*hook1_count.lock().unwrap(), 1); - - // Replace with second hook - let h2 = hook2_count.clone(); - cf.set_commit_hook(move |_ops, _commit_seq| { - *h2.lock().unwrap() += 1; - 0 - }) - .unwrap(); - - { - let mut txn = db.begin_transaction().unwrap(); - txn.put(&cf, b"key2", b"value2", -1).unwrap(); - txn.commit().unwrap(); - } - - // First hook should still be 1, second hook should be 1 - assert_eq!(*hook1_count.lock().unwrap(), 1); - assert_eq!(*hook2_count.lock().unwrap(), 1); - } - - #[test] - fn test_commit_hook_multi_op_transaction() { - use std::sync::{Arc, Mutex}; - - let (db, _temp_dir) = create_test_db(); - - let cf_config = ColumnFamilyConfig::default(); - db.create_column_family("test_cf", cf_config).unwrap(); - let mut cf = db.get_column_family("test_cf").unwrap(); - - let ops_count: Arc> = Arc::new(Mutex::new(0)); - let ops_count_clone = ops_count.clone(); - - cf.set_commit_hook(move |ops, _commit_seq| { - *ops_count_clone.lock().unwrap() += ops.len(); - 0 - }) - .unwrap(); - - // Commit transaction with multiple operations - { - let mut txn = db.begin_transaction().unwrap(); - txn.put(&cf, b"key1", b"value1", -1).unwrap(); - txn.put(&cf, b"key2", b"value2", -1).unwrap(); - txn.put(&cf, b"key3", b"value3", -1).unwrap(); - txn.commit().unwrap(); - } - - // Hook should have received all 3 operations - assert_eq!(*ops_count.lock().unwrap(), 3); - } - - #[test] - fn test_init_finalize() { - // init() may fail if already auto-initialized by another test, - // but finalize() should always be safe to call - let _ = init(); - finalize(); - } - - #[test] - fn test_max_memory_usage_config() { - let config = Config::new("./test_mem").max_memory_usage(512 * 1024 * 1024); // 512MB - assert_eq!(config.max_memory_usage, 512 * 1024 * 1024); - - // Default should be 0 (auto) - let default_config = Config::default(); - assert_eq!(default_config.max_memory_usage, 0); - } - - #[test] - fn test_delete_column_family() { - let (db, _temp_dir) = create_test_db(); - - let cf_config = ColumnFamilyConfig::default(); - db.create_column_family("to_delete", cf_config).unwrap(); - - // Insert some data - { - let cf = db.get_column_family("to_delete").unwrap(); - let mut txn = db.begin_transaction().unwrap(); - txn.put(&cf, b"key", b"value", -1).unwrap(); - txn.commit().unwrap(); - } - - // Get the column family and delete by pointer - let cf = db.get_column_family("to_delete").unwrap(); - db.delete_column_family(cf).unwrap(); - - // Verify it's gone - assert!(db.get_column_family("to_delete").is_err()); - } - - #[test] - fn test_register_custom_comparator() { - let (db, _temp_dir) = create_test_db(); - - // Register a reverse comparator - db.register_comparator("test_reverse", |key1, key2| { - let min_len = key1.len().min(key2.len()); - for i in 0..min_len { - if key1[i] != key2[i] { - return key2[i] as i32 - key1[i] as i32; - } - } - key2.len() as i32 - key1.len() as i32 - }) - .unwrap(); - - // Verify it's registered - assert!(db.has_comparator("test_reverse")); - - // Create a column family using this comparator - let cf_config = ColumnFamilyConfig::new().comparator_name("test_reverse"); - db.create_column_family("reverse_cf", cf_config).unwrap(); - - let cf = db.get_column_family("reverse_cf").unwrap(); - - // Insert data - { - let mut txn = db.begin_transaction().unwrap(); - txn.put(&cf, b"aaa", b"first", -1).unwrap(); - txn.put(&cf, b"zzz", b"last", -1).unwrap(); - txn.commit().unwrap(); - } - - // With reverse comparator, iterating forward should give zzz before aaa - { - let txn = db.begin_transaction().unwrap(); - let mut iter = txn.new_iterator(&cf).unwrap(); - iter.seek_to_first().unwrap(); - - assert!(iter.is_valid()); - let first_key = iter.key().unwrap(); - assert_eq!(first_key, b"zzz"); - - iter.next().unwrap(); - assert!(iter.is_valid()); - let second_key = iter.key().unwrap(); - assert_eq!(second_key, b"aaa"); - } - } - - #[test] - fn test_has_comparator_builtin() { - let (db, _temp_dir) = create_test_db(); - - // Built-in comparators should be registered - assert!(db.has_comparator("memcmp")); - assert!(db.has_comparator("reverse")); - assert!(db.has_comparator("lexicographic")); - assert!(db.has_comparator("uint64")); - assert!(db.has_comparator("int64")); - assert!(db.has_comparator("case_insensitive")); - - // Non-existent comparator - assert!(!db.has_comparator("nonexistent_comparator")); - } - - #[test] - fn test_block_based_vs_btree_config() { - let (db, _temp_dir) = create_test_db(); - - // Create block-based column family (default) - let block_config = ColumnFamilyConfig::new().use_btree(false); - db.create_column_family("block_cf", block_config).unwrap(); - - // Create B+tree column family - let btree_config = ColumnFamilyConfig::new().use_btree(true); - db.create_column_family("btree_cf", btree_config).unwrap(); - - // Verify both can be used - let block_cf = db.get_column_family("block_cf").unwrap(); - let btree_cf = db.get_column_family("btree_cf").unwrap(); - - // Insert data into both - { - let mut txn = db.begin_transaction().unwrap(); - txn.put(&block_cf, b"key1", b"value1", -1).unwrap(); - txn.put(&btree_cf, b"key1", b"value1", -1).unwrap(); - txn.commit().unwrap(); - } - - // Read from both - { - let txn = db.begin_transaction().unwrap(); - let v1 = txn.get(&block_cf, b"key1").unwrap(); - let v2 = txn.get(&btree_cf, b"key1").unwrap(); - assert_eq!(v1, b"value1"); - assert_eq!(v2, b"value1"); - } - } - - #[test] - fn test_purge_cf() { - let (db, _temp_dir) = create_test_db(); - - let cf_config = ColumnFamilyConfig::default(); - db.create_column_family("test_cf", cf_config).unwrap(); - let cf = db.get_column_family("test_cf").unwrap(); - - // Insert some data - { - let mut txn = db.begin_transaction().unwrap(); - for i in 0..50 { - let key = format!("key{:04}", i); - let value = format!("value{}", i); - txn.put(&cf, key.as_bytes(), value.as_bytes(), -1).unwrap(); - } - txn.commit().unwrap(); - } - - // Purge the column family (synchronous flush + compaction) - cf.purge().unwrap(); - - // Verify data is still accessible after purge - { - let txn = db.begin_transaction().unwrap(); - let value = txn.get(&cf, b"key0000").unwrap(); - assert_eq!(value, b"value0"); - let value = txn.get(&cf, b"key0049").unwrap(); - assert_eq!(value, b"value49"); - } - } - - #[test] - fn test_purge_db() { - let (db, _temp_dir) = create_test_db(); - - // Create multiple column families with data - for cf_name in &["cf_a", "cf_b"] { - let cf_config = ColumnFamilyConfig::default(); - db.create_column_family(cf_name, cf_config).unwrap(); - let cf = db.get_column_family(cf_name).unwrap(); - - let mut txn = db.begin_transaction().unwrap(); - for i in 0..20 { - let key = format!("key{:04}", i); - let value = format!("value{}", i); - txn.put(&cf, key.as_bytes(), value.as_bytes(), -1).unwrap(); - } - txn.commit().unwrap(); - } - - // Purge entire database - db.purge().unwrap(); - - // Verify data is still accessible after purge - for cf_name in &["cf_a", "cf_b"] { - let cf = db.get_column_family(cf_name).unwrap(); - let txn = db.begin_transaction().unwrap(); - let value = txn.get(&cf, b"key0000").unwrap(); - assert_eq!(value, b"value0"); - } - } - - #[test] - fn test_sync_wal() { - let (db, _temp_dir) = create_test_db(); - - let cf_config = ColumnFamilyConfig::new().sync_mode(SyncMode::None); - db.create_column_family("test_cf", cf_config).unwrap(); - let cf = db.get_column_family("test_cf").unwrap(); - - // Insert some data - { - let mut txn = db.begin_transaction().unwrap(); - txn.put(&cf, b"key1", b"value1", -1).unwrap(); - txn.put(&cf, b"key2", b"value2", -1).unwrap(); - txn.commit().unwrap(); - } - - // Force WAL sync - cf.sync_wal().unwrap(); - - // Verify data is still accessible - { - let txn = db.begin_transaction().unwrap(); - let value = txn.get(&cf, b"key1").unwrap(); - assert_eq!(value, b"value1"); - } - } - - #[test] - fn test_sync_wal_interval_mode() { - let (db, _temp_dir) = create_test_db(); - - let cf_config = ColumnFamilyConfig::new() - .sync_mode(SyncMode::Interval) - .sync_interval_us(1000000); - db.create_column_family("test_cf", cf_config).unwrap(); - let cf = db.get_column_family("test_cf").unwrap(); - - // Insert data and sync WAL - { - let mut txn = db.begin_transaction().unwrap(); - txn.put(&cf, b"key1", b"value1", -1).unwrap(); - txn.commit().unwrap(); - } - - // Force immediate WAL durability - cf.sync_wal().unwrap(); - } - - #[test] - fn test_get_db_stats() { - let (db, _temp_dir) = create_test_db(); - - let cf_config = ColumnFamilyConfig::default(); - db.create_column_family("test_cf", cf_config).unwrap(); - let cf = db.get_column_family("test_cf").unwrap(); - - // Insert some data - { - let mut txn = db.begin_transaction().unwrap(); - for i in 0..100 { - let key = format!("key{:04}", i); - let value = format!("value{}", i); - txn.put(&cf, key.as_bytes(), value.as_bytes(), -1).unwrap(); - } - txn.commit().unwrap(); - } - - let stats = db.get_db_stats().unwrap(); - - // Should have at least 1 column family - assert!(stats.num_column_families >= 1); - // Total memory should be non-zero - assert!(stats.total_memory > 0); - // Memory pressure should be in valid range (0-3) - assert!(stats.memory_pressure_level >= 0 && stats.memory_pressure_level <= 3); - } - - #[test] - fn test_get_db_stats_multiple_cfs() { - let (db, _temp_dir) = create_test_db(); - - // Create multiple column families - for cf_name in &["cf_1", "cf_2", "cf_3"] { - let cf_config = ColumnFamilyConfig::default(); - db.create_column_family(cf_name, cf_config).unwrap(); - } - - let stats = db.get_db_stats().unwrap(); - assert!(stats.num_column_families >= 3); - } - - #[test] - fn test_purge_cf_empty() { - let (db, _temp_dir) = create_test_db(); - - let cf_config = ColumnFamilyConfig::default(); - db.create_column_family("test_cf", cf_config).unwrap(); - let cf = db.get_column_family("test_cf").unwrap(); - - // Purge empty column family should succeed - cf.purge().unwrap(); - } - - #[test] - fn test_purge_db_empty() { - let (db, _temp_dir) = create_test_db(); - - // Purge database with no column families should succeed - db.purge().unwrap(); - } - - #[test] - fn test_unified_memtable_config() { - let temp_dir = TempDir::new().unwrap(); - let config = Config::new(temp_dir.path()) - .num_flush_threads(2) - .num_compaction_threads(2) - .log_level(LogLevel::Info) - .block_cache_size(64 * 1024 * 1024) - .max_open_sstables(256) - .unified_memtable(true) - .unified_memtable_write_buffer_size(128 * 1024 * 1024) - .unified_memtable_skip_list_max_level(16) - .unified_memtable_skip_list_probability(0.25) - .unified_memtable_sync_mode(SyncMode::None) - .unified_memtable_sync_interval_us(0); - - assert!(config.unified_memtable); - assert_eq!(config.unified_memtable_write_buffer_size, 128 * 1024 * 1024); - assert_eq!(config.unified_memtable_skip_list_max_level, 16); - - let db = TidesDB::open(config).unwrap(); - db.create_column_family("test_cf", ColumnFamilyConfig::default()) - .unwrap(); - let cf = db.get_column_family("test_cf").unwrap(); - - // Insert and read data to verify unified memtable works - { - let mut txn = db.begin_transaction().unwrap(); - txn.put(&cf, b"key1", b"value1", -1).unwrap(); - txn.commit().unwrap(); - } - - { - let txn = db.begin_transaction().unwrap(); - let value = txn.get(&cf, b"key1").unwrap(); - assert_eq!(value, b"value1"); - } - } - - #[test] - fn test_iterator_key_value() { - let (db, _temp_dir) = create_test_db(); - - let cf_config = ColumnFamilyConfig::default(); - db.create_column_family("test_cf", cf_config).unwrap(); - let cf = db.get_column_family("test_cf").unwrap(); - - // Insert some data - { - let mut txn = db.begin_transaction().unwrap(); - for i in 0..10 { - let key = format!("key{:02}", i); - let value = format!("value{}", i); - txn.put(&cf, key.as_bytes(), value.as_bytes(), -1).unwrap(); - } - txn.commit().unwrap(); - } - - // Use key_value() combined method - { - let txn = db.begin_transaction().unwrap(); - let mut iter = txn.new_iterator(&cf).unwrap(); - iter.seek_to_first().unwrap(); - - let mut count = 0; - while iter.is_valid() { - let (key, value) = iter.key_value().unwrap(); - assert!(!key.is_empty()); - assert!(!value.is_empty()); - count += 1; - iter.next().unwrap(); - } - assert_eq!(count, 10); - } - } - - #[test] - fn test_db_stats_unified_fields() { - let (db, _temp_dir) = create_test_db(); - - let cf_config = ColumnFamilyConfig::default(); - db.create_column_family("test_cf", cf_config).unwrap(); - - let stats = db.get_db_stats().unwrap(); - - // Unified memtable fields should be accessible - let _ = stats.unified_memtable_enabled; - let _ = stats.unified_memtable_bytes; - let _ = stats.unified_immutable_count; - let _ = stats.unified_is_flushing; - let _ = stats.unified_next_cf_index; - let _ = stats.unified_wal_generation; - - // Object store fields should be accessible - let _ = stats.object_store_enabled; - let _ = stats.object_store_connector; - let _ = stats.local_cache_bytes_used; - let _ = stats.local_cache_bytes_max; - let _ = stats.local_cache_num_files; - let _ = stats.last_uploaded_generation; - let _ = stats.upload_queue_depth; - let _ = stats.total_uploads; - let _ = stats.total_upload_failures; - let _ = stats.replica_mode; - - // Single-writer fencing epochs should be accessible. On a local - // (non-object-store) database they are 0; on libraries older than - // 9.3.8 they are reported as 0 as well. - let _ = stats.primary_epoch; - let _ = stats.seen_epoch; - - // For a non-unified, non-object-store db: - assert!(!stats.unified_memtable_enabled); - assert!(!stats.object_store_enabled); - assert!(!stats.replica_mode); - assert_eq!(stats.primary_epoch, 0); - assert_eq!(stats.seen_epoch, 0); - } - - #[test] - fn test_db_stats_unified_memtable_enabled() { - let temp_dir = TempDir::new().unwrap(); - let config = Config::new(temp_dir.path()) - .num_flush_threads(2) - .num_compaction_threads(2) - .log_level(LogLevel::Info) - .block_cache_size(64 * 1024 * 1024) - .max_open_sstables(256) - .unified_memtable(true); - - let db = TidesDB::open(config).unwrap(); - db.create_column_family("test_cf", ColumnFamilyConfig::default()) - .unwrap(); - - let stats = db.get_db_stats().unwrap(); - assert!(stats.unified_memtable_enabled); - } - - #[test] - fn test_cf_config_object_store_fields() { - let config = ColumnFamilyConfig::new() - .object_lazy_compaction(true) - .object_prefetch_compaction(false); - - assert!(config.object_lazy_compaction); - assert!(!config.object_prefetch_compaction); - } - - #[test] - fn test_error_code_readonly() { - // Verify ReadOnly error code is recognized - let code = ErrorCode::from_code(-13); - assert_eq!(code, Some(ErrorCode::ReadOnly)); - } - - #[test] - fn test_promote_to_primary_not_replica() { - let (db, _temp_dir) = create_test_db(); - - // promote_to_primary on a non-replica db should return an error - let result = db.promote_to_primary(); - assert!(result.is_err()); - } - - #[cfg(tidesdb_has_txn_single_delete)] - #[test] - fn test_transaction_single_delete() { - let (db, _temp_dir) = create_test_db(); - - let cf_config = ColumnFamilyConfig::default(); - db.create_column_family("test_cf", cf_config).unwrap(); - let cf = db.get_column_family("test_cf").unwrap(); - - // Put once, single-delete once (matches the single_delete contract) - { - let mut txn = db.begin_transaction().unwrap(); - txn.put(&cf, b"sd_key", b"sd_value", -1).unwrap(); - txn.commit().unwrap(); - } - - { - let txn = db.begin_transaction().unwrap(); - let value = txn.get(&cf, b"sd_key").unwrap(); - assert_eq!(value, b"sd_value"); - } - - { - let mut txn = db.begin_transaction().unwrap(); - txn.single_delete(&cf, b"sd_key").unwrap(); - txn.commit().unwrap(); - } - - // Verify the key is no longer visible - { - let txn = db.begin_transaction().unwrap(); - let result = txn.get(&cf, b"sd_key"); - assert!(result.is_err()); - } - } - - #[cfg(tidesdb_has_txn_single_delete)] - #[test] - fn test_transaction_single_delete_in_batch() { - let (db, _temp_dir) = create_test_db(); - - let cf_config = ColumnFamilyConfig::default(); - db.create_column_family("test_cf", cf_config).unwrap(); - let cf = db.get_column_family("test_cf").unwrap(); - - // Insert several keys - { - let mut txn = db.begin_transaction().unwrap(); - for i in 0..5 { - let key = format!("k{}", i); - let val = format!("v{}", i); - txn.put(&cf, key.as_bytes(), val.as_bytes(), -1).unwrap(); - } - txn.commit().unwrap(); - } - - // Mix put + single_delete in one batch (different keys) - { - let mut txn = db.begin_transaction().unwrap(); - txn.put(&cf, b"k_new", b"v_new", -1).unwrap(); - txn.single_delete(&cf, b"k0").unwrap(); - txn.single_delete(&cf, b"k1").unwrap(); - txn.commit().unwrap(); - } - - { - let txn = db.begin_transaction().unwrap(); - assert!(txn.get(&cf, b"k0").is_err()); - assert!(txn.get(&cf, b"k1").is_err()); - assert_eq!(txn.get(&cf, b"k2").unwrap(), b"v2"); - assert_eq!(txn.get(&cf, b"k_new").unwrap(), b"v_new"); - } - } - - #[cfg(tidesdb_has_tombstone_stats)] - #[test] - fn test_tombstone_cf_config_roundtrip() { - let (db, _temp_dir) = create_test_db(); - - let cf_config = ColumnFamilyConfig::new() - .tombstone_density_trigger(0.5) - .tombstone_density_min_entries(256); - - db.create_column_family("ts_cf", cf_config).unwrap(); - let cf = db.get_column_family("ts_cf").unwrap(); - - let stats = cf.get_stats().unwrap(); - let cfg = stats.config.expect("stats should expose CF config"); - assert_eq!(cfg.tombstone_density_trigger, 0.5); - assert_eq!(cfg.tombstone_density_min_entries, 256); - - // Library defaults should be sensible (min_entries ~1024). - let defaults = ColumnFamilyConfig::default(); - assert!( - defaults.tombstone_density_min_entries > 0, - "default min_entries should be sourced from C library" - ); - } - - #[cfg(tidesdb_has_tombstone_stats)] - #[test] - fn test_tombstone_stats_populated_after_deletes() { - let (db, _temp_dir) = create_test_db(); - - let cf_config = ColumnFamilyConfig::default(); - db.create_column_family("ts_stats_cf", cf_config).unwrap(); - let cf = db.get_column_family("ts_stats_cf").unwrap(); - - const N: usize = 200; - - { - let mut txn = db.begin_transaction().unwrap(); - for i in 0..N { - let key = format!("key{:04}", i); - let value = format!("value{}", i); - txn.put(&cf, key.as_bytes(), value.as_bytes(), -1).unwrap(); - } - txn.commit().unwrap(); - } - cf.flush_memtable().unwrap(); - - { - let mut txn = db.begin_transaction().unwrap(); - for i in 0..(N / 2) { - let key = format!("key{:04}", i); - txn.delete(&cf, key.as_bytes()).unwrap(); - } - txn.commit().unwrap(); - } - cf.flush_memtable().unwrap(); - - let mut stats = cf.get_stats().unwrap(); - for _ in 0..10 { - if stats.total_tombstones > 0 { - break; - } - std::thread::sleep(std::time::Duration::from_millis(100)); - stats = cf.get_stats().unwrap(); - } - - assert!( - stats.total_tombstones > 0, - "expected tombstones after deletes" - ); - assert!(stats.tombstone_ratio >= 0.0 && stats.tombstone_ratio <= 1.0); - assert!(stats.max_sst_density >= 0.0 && stats.max_sst_density <= 1.0); - assert_eq!( - stats.level_tombstone_counts.len(), - stats.num_levels as usize - ); - } - - #[cfg(tidesdb_has_compact_range)] - #[test] - fn test_compact_range_basic() { - let (db, _temp_dir) = create_test_db(); - - let cf_config = ColumnFamilyConfig::default(); - db.create_column_family("range_cf", cf_config).unwrap(); - let cf = db.get_column_family("range_cf").unwrap(); - - // Multiple flushed batches to create several SSTables. - for batch in 0..3 { - let mut txn = db.begin_transaction().unwrap(); - for i in 0..50 { - let key = format!("key{:02}{:04}", batch, i); - let value = format!("v{}", i); - txn.put(&cf, key.as_bytes(), value.as_bytes(), -1).unwrap(); - } - txn.commit().unwrap(); - cf.flush_memtable().unwrap(); - } - std::thread::sleep(std::time::Duration::from_millis(200)); - - // Narrow range over batch 1 only. - cf.compact_range(Some(b"key010000"), Some(b"key019999")) - .unwrap(); - - // Both endpoints unbounded should be rejected. - let err = cf.compact_range(None, None).unwrap_err(); - assert!(matches!( - err, - Error::TidesDB { - code: ErrorCode::InvalidArgs, - .. - } - )); - - // Empty slices should also be treated as unbounded -> rejected. - let err = cf.compact_range(Some(&[]), Some(&[])).unwrap_err(); - assert!(matches!( - err, - Error::TidesDB { - code: ErrorCode::InvalidArgs, - .. - } - )); - - // A key outside the compacted range remains readable and unchanged. - let txn = db.begin_transaction().unwrap(); - let val = txn.get(&cf, b"key020000").unwrap(); - assert_eq!(val, b"v0"); - } - - #[cfg(tidesdb_has_cancel_background_work)] - #[test] - fn test_cancel_background_work() { - let (db, _temp_dir) = create_test_db(); - db.cancel_background_work().unwrap(); - } - - #[cfg(tidesdb_has_raise_open_file_limit)] - #[test] - fn test_raise_open_file_limit_raises_or_preserves_ceiling() { - let current = raise_open_file_limit(0); - assert!(current > 0); - - let requested = current.saturating_add(1); - let raised = raise_open_file_limit(requested); - assert!(raised >= current); - } - - #[cfg(tidesdb_has_max_concurrent_flushes)] - #[test] - fn test_max_concurrent_flushes() { - let temp_dir = TempDir::new().unwrap(); - let config = Config::new(temp_dir.path()).max_concurrent_flushes(1); - assert_eq!(config.max_concurrent_flushes, 1); - - let db = TidesDB::open(config).unwrap(); - db.create_column_family("test_cf", ColumnFamilyConfig::default()) - .unwrap(); - let cf = db.get_column_family("test_cf").unwrap(); - - let mut txn = db.begin_transaction().unwrap(); - txn.put(&cf, b"k", b"v", -1).unwrap(); - txn.commit().unwrap(); - cf.flush_memtable().unwrap(); - - let txn = db.begin_transaction().unwrap(); - assert_eq!(txn.get(&cf, b"k").unwrap(), b"v"); - } - - #[test] - fn test_default_config_sources_max_concurrent_flushes() { - let cfg = Config::default(); - let c = unsafe { ffi::tidesdb_default_config() }; - #[cfg(tidesdb_has_max_concurrent_flushes)] - assert_eq!(cfg.max_concurrent_flushes, c.max_concurrent_flushes); - #[cfg(not(tidesdb_has_max_concurrent_flushes))] - assert_eq!(cfg.max_concurrent_flushes, 0); - } - - #[test] - fn test_error_code_busy() { - // Verify the Busy error code (-14) is recognized and rendered. - let code = ErrorCode::from_code(-14); - assert_eq!(code, Some(ErrorCode::Busy)); - assert_eq!(ErrorCode::Busy.to_string(), "resource is busy"); - } - - #[test] - fn test_error_code_precondition() { - // Verify the Precondition error code (-15) is recognized and rendered. - let code = ErrorCode::from_code(-15); - assert_eq!(code, Some(ErrorCode::Precondition)); - assert_eq!(ErrorCode::Precondition.to_string(), "precondition not met"); - } - - #[test] - fn test_builtin_comparator_symbols_callable() { - // The built-in comparator C functions should be linkable and behave like memcmp. - let a = b"abc"; - let b = b"abd"; - let lt = unsafe { - ffi::tidesdb_comparator_memcmp( - a.as_ptr(), - a.len(), - b.as_ptr(), - b.len(), - std::ptr::null_mut(), - ) - }; - assert!(lt < 0, "abc should sort before abd"); - - let eq = unsafe { - ffi::tidesdb_comparator_memcmp( - a.as_ptr(), - a.len(), - a.as_ptr(), - a.len(), - std::ptr::null_mut(), - ) - }; - assert_eq!(eq, 0, "equal keys should compare equal"); - - // case_insensitive should treat ABC == abc. - let upper = b"ABC"; - let ci = unsafe { - ffi::tidesdb_comparator_case_insensitive( - upper.as_ptr(), - upper.len(), - a.as_ptr(), - a.len(), - std::ptr::null_mut(), - ) - }; - assert_eq!(ci, 0, "case-insensitive compare of ABC and abc should be equal"); - } - - #[cfg(tidesdb_has_write_amp_stats)] - #[test] - fn test_write_amp_cf_stats() { - let (db, _temp_dir) = create_test_db(); - - let cf_config = ColumnFamilyConfig::default(); - db.create_column_family("wa_cf", cf_config).unwrap(); - let cf = db.get_column_family("wa_cf").unwrap(); - - { - let mut txn = db.begin_transaction().unwrap(); - for i in 0..200 { - let key = format!("key{:04}", i); - let value = format!("value{}", i); - txn.put(&cf, key.as_bytes(), value.as_bytes(), -1).unwrap(); - } - txn.commit().unwrap(); - } - cf.flush_memtable().unwrap(); - cf.purge().unwrap(); - - let stats = cf.get_stats().unwrap(); - - // Write-amplification counters should be accessible. - let _ = stats.compaction_bytes_written; - let _ = stats.compaction_bytes_read; - let _ = stats.flush_count; - let _ = stats.compaction_count; - - // Committing real data must register user bytes and at least one flush. - assert!( - stats.user_bytes_written > 0, - "expected user_bytes_written after committing data" - ); - assert!( - stats.flush_bytes_written > 0, - "expected flush_bytes_written after flushing" - ); - } - - #[cfg(tidesdb_has_write_amp_stats)] - #[test] - fn test_write_amp_db_stats() { - let (db, _temp_dir) = create_test_db(); - - let cf_config = ColumnFamilyConfig::default(); - db.create_column_family("wa_db_cf", cf_config).unwrap(); - let cf = db.get_column_family("wa_db_cf").unwrap(); - - { - let mut txn = db.begin_transaction().unwrap(); - for i in 0..200 { - let key = format!("key{:04}", i); - let value = format!("value{}", i); - txn.put(&cf, key.as_bytes(), value.as_bytes(), -1).unwrap(); - } - txn.commit().unwrap(); - } - cf.flush_memtable().unwrap(); - - let stats = db.get_db_stats().unwrap(); - - // Write-amplification counters should be accessible. - let _ = stats.uwal_bytes_written; - let _ = stats.wal_bytes_written; - let _ = stats.flush_bytes_written; - let _ = stats.compaction_bytes_written; - let _ = stats.compaction_bytes_read; - let _ = stats.flush_count; - let _ = stats.compaction_count; - - assert!( - stats.user_bytes_written > 0, - "expected db-wide user_bytes_written after committing data" - ); - } - - #[cfg(feature = "objectstore")] - #[test] - fn test_s3_config_builder() { - let s3 = S3Config::new("minio.local:9000", "mybucket", "ak", "sk") - .prefix("db1/") - .region("us-east-1") - .use_ssl(false) - .use_path_style(true) - .multipart_threshold(64 * 1024 * 1024); - - assert_eq!(s3.endpoint, "minio.local:9000"); - assert_eq!(s3.bucket, "mybucket"); - assert_eq!(s3.prefix.as_deref(), Some("db1/")); - assert_eq!(s3.region.as_deref(), Some("us-east-1")); - assert!(!s3.use_ssl); - assert!(s3.use_path_style); - assert_eq!(s3.multipart_threshold, 64 * 1024 * 1024); - - let config = Config::new("./s3db").object_store_s3(s3); - assert!(config.object_store_s3.is_some()); - } -} +pub use stats::{ + CacheStats, CfStats, DbStats, EncodingStats, IoStat, IoStats, RangeStats, StallStat, StallStats, +}; +pub use transaction::{AbortHandle, Transaction, TxnState}; + +/// The TidesDB C library release this crate binds and bundles. +/// +/// The crate's major version maps to a TidesDB major: crate N.x binds TidesDB (N+9).x, so +/// crate 1.x binds TidesDB 10.x and crate 2.x will bind TidesDB 11.x. +pub const TIDESDB_VERSION: &str = ffi::TIDESDB_VERSION; + +/// The TidesDB release actually linked: [`TIDESDB_VERSION`] when the bundled source was built, +/// or the version of a system TidesDB found through pkg-config (same major, at least +/// [`TIDESDB_VERSION`]). +pub const TIDESDB_LINKED_VERSION: &str = env!("TIDESDB_LINKED_VERSION"); diff --git a/src/stats.rs b/src/stats.rs index 1e41812..64eeb6d 100644 --- a/src/stats.rs +++ b/src/stats.rs @@ -6,187 +6,446 @@ //! Statistics types for TidesDB. use crate::config::ColumnFamilyConfig; +use crate::ffi; +use std::ffi::CStr; -/// Statistics for a column family. +/// Per-column-family statistics, returned by +/// [`ColumnFamily::get_stats`](crate::ColumnFamily::get_stats). +/// +/// Memtable figures are database-level and live in [`DbStats`]. #[derive(Debug, Clone)] -pub struct Stats { - /// Number of levels +pub struct CfStats { + /// The family's current name. Unlike [`ColumnFamily::name`](crate::ColumnFamily::name), + /// this reflects a rename made after the handle was looked up. + pub name: String, + /// Deepest level that currently holds an sstable (0 for a family with nothing flushed) pub num_levels: i32, - /// Memtable size in bytes - pub memtable_size: usize, - /// Size of each level in bytes + /// A copy of the column family configuration + pub config: ColumnFamilyConfig, + /// On-disk size of each level, index 0 is L1 pub level_sizes: Vec, - /// Number of SSTables in each level + /// SSTable count of each level pub level_num_sstables: Vec, - /// Column family configuration (if available) - pub config: Option, - /// Total number of keys across memtable and all sstables + /// Key count of each level + pub level_key_counts: Vec, + /// Tombstone count of each level + pub level_tombstone_counts: Vec, + /// Sum of each sstable's distinct key count (a key present in several sstables counts + /// once per sstable) pub total_keys: u64, - /// Total data size (klog + vlog) across all sstables + /// The family's own on-disk size (sum of its key logs); separated values are counted + /// in [`DbStats::vlog_file_size`] pub total_data_size: u64, - /// Average key size in bytes + /// Average key length in bytes pub avg_key_size: f64, - /// Average value size in bytes + /// Average value length in bytes pub avg_value_size: f64, - /// Number of keys per level - pub level_key_counts: Vec, - /// Read amplification (point lookup cost multiplier) + /// Point-lookup read amplification (sstables a worst-case get may probe) pub read_amp: f64, - /// Cache hit rate (0.0 if cache disabled) - pub hit_rate: f64, - /// Whether column family uses B+tree format - pub use_btree: bool, - /// Total B+tree nodes across all SSTables (only populated if use_btree=true) + /// Total btree nodes pub btree_total_nodes: u64, - /// Maximum tree height across all SSTables (only populated if use_btree=true) + /// Maximum btree height pub btree_max_height: u32, - /// Average tree height across all SSTables (only populated if use_btree=true) + /// Average btree height pub btree_avg_height: f64, - /// Sum of `tombstone_count` across every SSTable in the column family + /// Total tombstones across every sstable pub total_tombstones: u64, - /// `total_tombstones / total_keys` (0.0 if `total_keys == 0`); range `[0.0, 1.0]` + /// `total_tombstones / total_keys` pub tombstone_ratio: f64, - /// Per-level tombstone counts (parallels `level_key_counts`) - pub level_tombstone_counts: Vec, - /// Worst per-SSTable tombstone density observed in the column family; range `[0.0, 1.0]` + /// Worst per-sstable tombstone density observed pub max_sst_density: f64, - /// 1-based level index where `max_sst_density` was observed (0 if no SSTables) + /// 1-based level of `max_sst_density`, 0 if none pub max_sst_density_level: i32, - /// Framed bytes appended to this CF's WAL (0 in unified mode). - /// Requires tidesdb >= 9.3.4; reported as 0 on older libraries. + /// This family's attributed share of write-ahead log bytes pub wal_bytes_written: u64, - /// On-disk bytes this CF's flushes wrote to L0 SSTables. - /// Requires tidesdb >= 9.3.4; reported as 0 on older libraries. + /// On-disk bytes this family's flushes wrote pub flush_bytes_written: u64, - /// On-disk bytes this CF's compactions wrote. - /// Requires tidesdb >= 9.3.4; reported as 0 on older libraries. + /// On-disk bytes this family's compactions wrote pub compaction_bytes_written: u64, - /// On-disk bytes this CF's compactions read as input. - /// Requires tidesdb >= 9.3.4; reported as 0 on older libraries. + /// On-disk bytes this family's compactions read pub compaction_bytes_read: u64, - /// Logical key+value bytes committed to this CF (write-amplification denominator). - /// Requires tidesdb >= 9.3.4; reported as 0 on older libraries. + /// Logical key+value bytes committed to this family pub user_bytes_written: u64, - /// Flushed SSTables produced by this CF. - /// Requires tidesdb >= 9.3.4; reported as 0 on older libraries. - pub flush_count: u64, - /// Compaction output SSTables produced by this CF. - /// Requires tidesdb >= 9.3.4; reported as 0 on older libraries. + /// Compactions this family has run pub compaction_count: u64, + /// Distinct keys still only in the shared memtables + pub unflushed_key_count: u64, + /// Memory held by this family's bloom filter directories outside the block cache + pub filter_resident_bytes: u64, } -/// Database-level aggregate statistics. +impl CfStats { + pub(crate) fn from_c(c: &ffi::tidesdb_cf_stats_t) -> Self { + let n = (c.num_levels.max(0) as usize).min(ffi::TDB_MAX_LEVELS); + // NUL-terminated within the fixed buffer; bounded in case it is not + let name_bytes: Vec = c + .config + .name + .iter() + .take_while(|&&b| b != 0) + .map(|&b| b as u8) + .collect(); + CfStats { + name: String::from_utf8_lossy(&name_bytes).into_owned(), + num_levels: c.num_levels, + config: ColumnFamilyConfig::from_c_config(&c.config), + level_sizes: c.level_sizes[..n].to_vec(), + level_num_sstables: c.level_num_sstables[..n].to_vec(), + level_key_counts: c.level_key_counts[..n].to_vec(), + level_tombstone_counts: c.level_tombstone_counts[..n].to_vec(), + total_keys: c.total_keys, + total_data_size: c.total_data_size, + avg_key_size: c.avg_key_size, + avg_value_size: c.avg_value_size, + read_amp: c.read_amp, + btree_total_nodes: c.btree_total_nodes, + btree_max_height: c.btree_max_height, + btree_avg_height: c.btree_avg_height, + total_tombstones: c.total_tombstones, + tombstone_ratio: c.tombstone_ratio, + max_sst_density: c.max_sst_density, + max_sst_density_level: c.max_sst_density_level, + wal_bytes_written: c.wal_bytes_written, + flush_bytes_written: c.flush_bytes_written, + compaction_bytes_written: c.compaction_bytes_written, + compaction_bytes_read: c.compaction_bytes_read, + user_bytes_written: c.user_bytes_written, + compaction_count: c.compaction_count, + unflushed_key_count: c.unflushed_key_count, + filter_resident_bytes: c.filter_resident_bytes, + } + } +} + +/// Database-level statistics, returned by [`TidesDB::get_db_stats`](crate::TidesDB::get_db_stats). #[derive(Debug, Clone)] pub struct DbStats { /// Number of column families pub num_column_families: i32, - /// System total memory - pub total_memory: u64, - /// System available memory at open time - pub available_memory: u64, - /// Resolved memory limit (auto or configured) - pub resolved_memory_limit: usize, - /// Current memory pressure (0=normal, 1=elevated, 2=high, 3=critical) - pub memory_pressure_level: i32, - /// Number of pending flush operations (queued + in-flight) - pub flush_pending_count: i32, - /// Total bytes in active memtables across all CFs - pub total_memtable_bytes: i64, - /// Total immutable memtables across all CFs - pub total_immutable_count: i32, - /// Total SSTables across all CFs and levels + /// Immutable memtables awaiting flush (the L0 queue depth) + pub immutable_memtable_count: i32, + /// Compaction jobs queued + pub compaction_pending_count: i32, + /// Total sstables across every column family and level pub total_sstable_count: i32, - /// Total data size (klog + vlog) across all CFs + /// On-disk key-log bytes across every column family pub total_data_size_bytes: u64, - /// Number of currently open SSTable file handles + /// Currently open sstable file handles pub num_open_sstables: i32, - /// Current global sequence number + /// Current global sequence number (the MVCC clock) pub global_seq: u64, + /// Oldest frozen snapshot any live transaction holds + pub min_snapshot_seq: u64, + /// Live registered transactions (read committed and stronger) + pub active_txn_count: i32, /// Bytes held by in-flight transactions pub txn_memory_bytes: i64, - /// Number of pending compaction tasks - pub compaction_queue_size: usize, - /// Number of pending flush tasks in queue - pub flush_queue_size: usize, - /// Whether unified memtable mode is active - pub unified_memtable_enabled: bool, - /// Bytes in unified active memtable - pub unified_memtable_bytes: i64, - /// Number of unified immutable memtables - pub unified_immutable_count: i32, - /// Whether unified memtable is currently flushing/rotating - pub unified_is_flushing: bool, - /// Next CF index to be assigned in unified mode - pub unified_next_cf_index: u32, - /// Current unified WAL generation counter - pub unified_wal_generation: u64, - /// Whether object store mode is active - pub object_store_enabled: bool, - /// Connector name ("s3", "gcs", "fs", etc.) - pub object_store_connector: String, - /// Current local file cache usage in bytes - pub local_cache_bytes_used: usize, - /// Configured maximum local cache size in bytes - pub local_cache_bytes_max: usize, - /// Number of files tracked in local cache - pub local_cache_num_files: i32, - /// Highest WAL generation confirmed uploaded - pub last_uploaded_generation: u64, - /// Number of pending upload jobs in the queue - pub upload_queue_depth: usize, - /// Lifetime count of objects uploaded to object store - pub total_uploads: u64, - /// Lifetime count of permanently failed uploads (after all retries) - pub total_upload_failures: u64, - /// Whether running in read-only replica mode - pub replica_mode: bool, - /// Single-writer fencing: lease epoch this primary currently holds (0 when not a - /// primary / no lease). Requires tidesdb >= 9.3.8; reported as 0 on older libraries. - pub primary_epoch: u64, - /// Single-writer fencing: highest lease epoch a replica has observed. - /// Requires tidesdb >= 9.3.8; reported as 0 on older libraries. - pub seen_epoch: u64, - /// Framed bytes appended to the shared unified WAL (0 if unified mode off). - /// Requires tidesdb >= 9.3.4; reported as 0 on older libraries. - pub uwal_bytes_written: u64, - /// Per-CF WAL bytes summed across all column families. - /// Requires tidesdb >= 9.3.4; reported as 0 on older libraries. - pub wal_bytes_written: u64, - /// Flush output bytes summed across all column families. - /// Requires tidesdb >= 9.3.4; reported as 0 on older libraries. + /// Transactions committed since open + pub txn_commits: u64, + /// Commits and prepares refused with a conflict since open + pub txn_conflicts: u64, + /// Memory the active memtable occupies + pub memtable_bytes: usize, + /// Whether an immutable memtable is queued or flushing + pub is_flushing: bool, + /// Next column family id to be assigned + pub next_cf_index: u32, + /// Generation of the active write-ahead log + pub wal_generation: u64, + /// SSTables flushed across every column family + pub flush_count: u64, + /// Compactions run across every column family + pub compaction_count: u64, + /// Flush output bytes pub flush_bytes_written: u64, - /// Compaction output bytes summed across all column families. - /// Requires tidesdb >= 9.3.4; reported as 0 on older libraries. + /// Compaction output bytes pub compaction_bytes_written: u64, - /// Compaction input bytes summed across all column families. - /// Requires tidesdb >= 9.3.4; reported as 0 on older libraries. + /// Compaction input bytes pub compaction_bytes_read: u64, - /// Logical committed bytes summed across all column families. - /// Requires tidesdb >= 9.3.4; reported as 0 on older libraries. + /// Framed write-ahead log bytes, measured at the append + pub wal_bytes_written: u64, + /// Logical committed bytes pub user_bytes_written: u64, - /// Flushed SSTables summed across all column families. - /// Requires tidesdb >= 9.3.4; reported as 0 on older libraries. - pub flush_count: u64, - /// Compaction output SSTables summed across all column families. - /// Requires tidesdb >= 9.3.4; reported as 0 on older libraries. - pub compaction_count: u64, + /// Total value-log file size + pub vlog_file_size: u64, + /// Values currently indexed in the value log + pub vlog_value_count: u64, + /// Uncompressed length the indexed values represent + pub vlog_used_bytes: u64, + /// On-disk length of the indexed values + pub vlog_stored_bytes: u64, + /// Value-log bytes the live sstables still reference + pub vlog_live_bytes: u64, + /// Open value-log segment files + pub vlog_segment_count: u64, + /// Value-log bytes ever appended, reclamation rewrites included + pub vlog_bytes_written: u64, + /// Value-log bytes beyond what live values account for + pub vlog_dead_bytes: u64, + /// Value-log reclaims attempted + pub vlog_reclaim_calls: u64, + /// Value-log reclaim passes that drained a segment + pub vlog_reclaim_passes: u64, + /// Value-log segment files a reclaim has unlinked + pub vlog_segments_retired: u64, + /// Sealed segments worth draining + pub vlog_segments_drainable: u64, + /// Commits the L0 admission policy made dwell + pub writes_throttled: u64, + /// Commits the L0 admission policy made wait for the flush queue + pub writes_blocked: u64, + /// Total microseconds commits were held in L0 admission + pub write_stall_us: u64, + /// Commits admitted only because the admission wait ceiling expired + pub write_stall_ceiling_hits: u64, } -/// Statistics for the block cache. +impl DbStats { + pub(crate) fn from_c(c: &ffi::tidesdb_db_stats_t) -> Self { + DbStats { + num_column_families: c.num_column_families, + immutable_memtable_count: c.immutable_memtable_count, + compaction_pending_count: c.compaction_pending_count, + total_sstable_count: c.total_sstable_count, + total_data_size_bytes: c.total_data_size_bytes, + num_open_sstables: c.num_open_sstables, + global_seq: c.global_seq, + min_snapshot_seq: c.min_snapshot_seq, + active_txn_count: c.active_txn_count, + txn_memory_bytes: c.txn_memory_bytes, + txn_commits: c.txn_commits, + txn_conflicts: c.txn_conflicts, + memtable_bytes: c.memtable_bytes, + is_flushing: c.is_flushing != 0, + next_cf_index: c.next_cf_index, + wal_generation: c.wal_generation, + flush_count: c.flush_count, + compaction_count: c.compaction_count, + flush_bytes_written: c.flush_bytes_written, + compaction_bytes_written: c.compaction_bytes_written, + compaction_bytes_read: c.compaction_bytes_read, + wal_bytes_written: c.wal_bytes_written, + user_bytes_written: c.user_bytes_written, + vlog_file_size: c.vlog_file_size, + vlog_value_count: c.vlog_value_count, + vlog_used_bytes: c.vlog_used_bytes, + vlog_stored_bytes: c.vlog_stored_bytes, + vlog_live_bytes: c.vlog_live_bytes, + vlog_segment_count: c.vlog_segment_count, + vlog_bytes_written: c.vlog_bytes_written, + vlog_dead_bytes: c.vlog_dead_bytes, + vlog_reclaim_calls: c.vlog_reclaim_calls, + vlog_reclaim_passes: c.vlog_reclaim_passes, + vlog_segments_retired: c.vlog_segments_retired, + vlog_segments_drainable: c.vlog_segments_drainable, + writes_throttled: c.writes_throttled, + writes_blocked: c.writes_blocked, + write_stall_us: c.write_stall_us, + write_stall_ceiling_hits: c.write_stall_ceiling_hits, + } + } +} + +/// Block cache statistics, returned by [`TidesDB::get_cache_stats`](crate::TidesDB::get_cache_stats). #[derive(Debug, Clone)] pub struct CacheStats { /// Whether the cache is enabled pub enabled: bool, - /// Total number of entries in the cache + /// Number of cached entries pub total_entries: usize, - /// Total bytes used by the cache + /// Bytes used by the cache pub total_bytes: usize, - /// Number of cache hits - pub hits: usize, - /// Number of cache misses - pub misses: usize, - /// Cache hit rate (0.0 to 1.0) + /// Cache hits + pub hits: u64, + /// Cache misses + pub misses: u64, + /// `hits / (hits + misses)` pub hit_rate: f64, - /// Number of cache partitions + /// Number of cache shards pub num_partitions: usize, } + +/// What one encoding chain achieved on the data it wrote, returned by +/// [`TidesDB::get_klog_encoding_stats`](crate::TidesDB::get_klog_encoding_stats) and +/// [`TidesDB::get_vlog_encoding_stats`](crate::TidesDB::get_vlog_encoding_stats). +#[derive(Debug, Clone)] +pub struct EncodingStats { + /// Encoding ids in the order applied; empty when stored verbatim. Ids `1..=4` are + /// [`CompressionAlgorithm`](crate::CompressionAlgorithm) values. + pub ids: Vec, + /// Size of the data before encoding + pub logical_bytes: u64, + /// Size of the data on disk + pub stored_bytes: u64, + /// Values for the value log, sstables for the key logs + pub item_count: u64, +} + +impl EncodingStats { + pub(crate) fn from_c(c: &ffi::tidesdb_encoding_stats_t) -> Self { + let n = (c.id_count.max(0) as usize).min(ffi::TDB_ENCODING_PIPELINE_MAX); + EncodingStats { + ids: c.ids[..n].to_vec(), + logical_bytes: c.logical_bytes, + stored_bytes: c.stored_bytes, + item_count: c.item_count, + } + } + + /// The realised ratio, `logical_bytes / stored_bytes` (0 when nothing is stored). + pub fn ratio(&self) -> f64 { + if self.stored_bytes == 0 { + 0.0 + } else { + self.logical_bytes as f64 / self.stored_bytes as f64 + } + } +} + +/// How much waiting one stall reason accounted for since the database opened. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct StallStat { + /// How many times a thread waited here + pub count: u64, + /// The summed wait in microseconds + pub total_us: u64, + /// The longest single wait in microseconds + pub max_us: u64, +} + +impl From for StallStat { + fn from(c: ffi::tidesdb_stall_stat_t) -> Self { + StallStat { + count: c.count, + total_us: c.total_us, + max_us: c.max_us, + } + } +} + +/// Where writers have been made to wait, returned by +/// [`TidesDB::get_stall_stats`](crate::TidesDB::get_stall_stats). +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct StallStats { + /// Waiting on the write-ahead log (staging space or, under a syncing mode, the write) + pub wal_append: StallStat, + /// Waiting to take the rotation lock while another committer rotated + pub rotate_lock: StallStat, + /// Performing a memtable rotation on everyone's behalf + pub rotate_work: StallStat, + /// Held by write admission because the unflushed backlog was too deep + pub admission: StallStat, + /// Inside a manifest commit + pub manifest_commit: StallStat, +} + +impl StallStats { + pub(crate) fn from_c(c: &ffi::tidesdb_stall_stats_t) -> Self { + StallStats { + wal_append: c.reasons[ffi::TDB_STALL_WAL_APPEND as usize].into(), + rotate_lock: c.reasons[ffi::TDB_STALL_ROTATE_LOCK as usize].into(), + rotate_work: c.reasons[ffi::TDB_STALL_ROTATE_WORK as usize].into(), + admission: c.reasons[ffi::TDB_STALL_ADMISSION as usize].into(), + manifest_commit: c.reasons[ffi::TDB_STALL_MANIFEST_COMMIT as usize].into(), + } + } + + /// Every reason with the library's stable short name, for logging or a stats table. + pub fn iter(&self) -> impl std::iter::Iterator { + [ + (ffi::TDB_STALL_WAL_APPEND, self.wal_append), + (ffi::TDB_STALL_ROTATE_LOCK, self.rotate_lock), + (ffi::TDB_STALL_ROTATE_WORK, self.rotate_work), + (ffi::TDB_STALL_ADMISSION, self.admission), + (ffi::TDB_STALL_MANIFEST_COMMIT, self.manifest_commit), + ] + .into_iter() + .map(|(reason, stat)| { + ( + static_name(unsafe { ffi::tidesdb_stall_reason_name(reason) }), + stat, + ) + }) + } +} + +/// What one class of file asked of the device. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct IoStat { + /// Writes issued + pub ops: u64, + /// Bytes written + pub bytes: u64, + /// Summed time inside those writes, in microseconds + pub total_us: u64, + /// The slowest single write, in microseconds + pub max_us: u64, +} + +impl From for IoStat { + fn from(c: ffi::tidesdb_io_stat_t) -> Self { + IoStat { + ops: c.ops, + bytes: c.bytes, + total_us: c.total_us, + max_us: c.max_us, + } + } +} + +/// Device writes per file class, returned by [`TidesDB::get_io_stats`](crate::TidesDB::get_io_stats). +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct IoStats { + /// Key logs, written by flush and compaction + pub sstable: IoStat, + /// The write-ahead log + pub wal: IoStat, + /// Value log segments + pub vlog: IoStat, +} + +impl IoStats { + pub(crate) fn from_c(c: &ffi::tidesdb_io_stats_t) -> Self { + IoStats { + sstable: c.classes[ffi::TDB_IO_SSTABLE as usize].into(), + wal: c.classes[ffi::TDB_IO_WAL as usize].into(), + vlog: c.classes[ffi::TDB_IO_VLOG as usize].into(), + } + } + + /// Every class with the library's stable short name, for logging or a stats table. + pub fn iter(&self) -> impl std::iter::Iterator { + [ + (ffi::TDB_IO_SSTABLE, self.sstable), + (ffi::TDB_IO_WAL, self.wal), + (ffi::TDB_IO_VLOG, self.vlog), + ] + .into_iter() + .map(|(cls, stat)| { + ( + static_name(unsafe { ffi::tidesdb_io_class_name(cls) }), + stat, + ) + }) + } +} + +/// Query-planner statistics for a key range, returned by +/// [`ColumnFamily::range_stats`](crate::ColumnFamily::range_stats). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RangeStats { + /// Sorted runs a scan of the range would merge + pub sstables_overlapping: u64, + /// Live keys the range holds + pub estimated_keys: u64, + /// Whether `estimated_keys` was counted exactly rather than estimated + pub keys_exact: bool, +} + +/// The C name functions return static literals that are never NULL and never freed. +fn static_name(ptr: *const libc::c_char) -> &'static str { + if ptr.is_null() { + return "unknown"; + } + unsafe { CStr::from_ptr(ptr) }.to_str().unwrap_or("unknown") +} diff --git a/src/tests.rs b/src/tests.rs new file mode 100644 index 0000000..eab5407 --- /dev/null +++ b/src/tests.rs @@ -0,0 +1,1651 @@ +// Package tidesdb +// Copyright (C) TidesDB +// +// Licensed under the Mozilla Public License, v. 2.0 (the "License"); + +use super::*; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use tempfile::TempDir; + +fn test_config(dir: &TempDir) -> Config { + Config::new(dir.path()) + .num_flush_threads(2) + .num_compaction_threads(2) + .log_level(LogLevel::Warn) + .block_cache_size(16 * 1024 * 1024) + .max_open_sstables(256) +} + +fn create_test_db() -> (TidesDB, TempDir) { + let temp_dir = TempDir::new().unwrap(); + let db = TidesDB::open(test_config(&temp_dir)).unwrap(); + (db, temp_dir) +} + +fn create_cf<'db>(db: &'db TidesDB, name: &str) -> ColumnFamily<'db> { + db.create_column_family(name, ColumnFamilyConfig::default()) + .unwrap(); + db.get_column_family(name).unwrap() +} + +fn put_all(db: &TidesDB, cf: &ColumnFamily<'_>, pairs: &[(&[u8], &[u8])]) { + let mut txn = db.begin_transaction().unwrap(); + for (k, v) in pairs { + txn.put(cf, k, v).unwrap(); + } + txn.commit().unwrap(); +} + +fn collect_keys(txn: &Transaction<'_>, cf: &ColumnFamily<'_>) -> Vec> { + let mut iter = txn.new_iterator(cf).unwrap(); + iter.seek_to_first().unwrap(); + let mut keys = Vec::new(); + while iter.is_valid() { + keys.push(iter.key().unwrap()); + iter.next().unwrap(); + } + keys +} + +fn now_secs() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs() as i64 +} + +// ===== lifecycle and column families ===== + +#[test] +fn test_version_constant() { + assert_eq!(TIDESDB_VERSION, "10.2.1"); +} + +#[test] +fn test_open_close() { + let (db, _dir) = create_test_db(); + drop(db); +} + +#[test] +fn test_second_open_of_same_dir_is_locked() { + let (_db, dir) = create_test_db(); + let err = TidesDB::open(test_config(&dir)).err().unwrap(); + assert_eq!(err.code(), Some(ErrorCode::Locked)); +} + +#[test] +fn test_reopen_persists_data() { + let dir = TempDir::new().unwrap(); + { + let db = TidesDB::open(test_config(&dir)).unwrap(); + let cf = create_cf(&db, "cf"); + put_all(&db, &cf, &[(b"k", b"v")]); + } + let db = TidesDB::open(test_config(&dir)).unwrap(); + let cf = db.get_column_family("cf").unwrap(); + let txn = db.begin_transaction().unwrap(); + assert_eq!(txn.get(&cf, b"k").unwrap(), b"v"); +} + +#[test] +fn test_create_list_drop_column_family() { + let (mut db, _dir) = create_test_db(); + { + let cf = create_cf(&db, "test_cf"); + assert_eq!(cf.name(), "test_cf"); + } + assert!( + db.list_column_families() + .unwrap() + .contains(&"test_cf".to_string()) + ); + + db.drop_column_family("test_cf").unwrap(); + assert!( + !db.list_column_families() + .unwrap() + .contains(&"test_cf".to_string()) + ); + assert!(db.get_column_family("test_cf").unwrap_err().is_not_found()); +} + +#[test] +fn test_create_duplicate_column_family() { + let (db, _dir) = create_test_db(); + create_cf(&db, "dup"); + let err = db + .create_column_family("dup", ColumnFamilyConfig::default()) + .unwrap_err(); + assert_eq!(err.code(), Some(ErrorCode::Exists)); +} + +#[test] +fn test_drop_column_family_unchecked() { + let (db, _dir) = create_test_db(); + create_cf(&db, "shared"); + unsafe { db.drop_column_family_unchecked("shared").unwrap() }; + assert!(db.get_column_family("shared").is_err()); +} + +#[test] +fn test_rename_column_family_keeps_handle_valid() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "old"); + put_all(&db, &cf, &[(b"k", b"v")]); + + db.rename_column_family("old", "new").unwrap(); + assert!(db.get_column_family("old").is_err()); + + // the old handle still refers to the same family + let txn = db.begin_transaction().unwrap(); + assert_eq!(txn.get(&cf, b"k").unwrap(), b"v"); + let renamed = db.get_column_family("new").unwrap(); + assert_eq!(txn.get(&renamed, b"k").unwrap(), b"v"); + + // the handle keeps its looked-up name; the stats report the current one + assert_eq!(cf.name(), "old"); + assert_eq!(cf.get_stats().unwrap().name, "new"); +} + +#[test] +fn test_clone_column_family() { + let (db, _dir) = create_test_db(); + let src = create_cf(&db, "src"); + put_all(&db, &src, &[(b"a", b"1"), (b"b", b"2")]); + + db.clone_column_family("src", "dst").unwrap(); + let dst = db.get_column_family("dst").unwrap(); + + // later writes to the source are not in the clone + put_all(&db, &src, &[(b"c", b"3")]); + + let txn = db.begin_transaction().unwrap(); + assert_eq!(txn.get(&dst, b"a").unwrap(), b"1"); + assert_eq!(txn.get(&dst, b"b").unwrap(), b"2"); + assert!(txn.get(&dst, b"c").unwrap_err().is_not_found()); +} + +#[test] +fn test_clone_column_family_errors() { + let (db, _dir) = create_test_db(); + create_cf(&db, "a"); + create_cf(&db, "b"); + assert!( + db.clone_column_family("missing", "x") + .unwrap_err() + .is_not_found() + ); + assert_eq!( + db.clone_column_family("a", "b").unwrap_err().code(), + Some(ErrorCode::Exists) + ); +} + +#[test] +fn test_cf_from_other_database_is_rejected() { + let (db1, _d1) = create_test_db(); + let (db2, _d2) = create_test_db(); + let cf2 = create_cf(&db2, "cf"); + let txn = db1.begin_transaction().unwrap(); + let err = txn.put(&cf2, b"k", b"v").unwrap_err(); + assert_eq!(err.code(), Some(ErrorCode::InvalidArgs)); +} + +// ===== reads and writes ===== + +#[test] +fn test_put_get_delete() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + + put_all(&db, &cf, &[(b"key", b"value")]); + assert_eq!( + db.begin_transaction().unwrap().get(&cf, b"key").unwrap(), + b"value" + ); + + let mut txn = db.begin_transaction().unwrap(); + txn.delete(&cf, b"key").unwrap(); + txn.commit().unwrap(); + + let txn = db.begin_transaction().unwrap(); + assert!(txn.get(&cf, b"key").unwrap_err().is_not_found()); + assert!(!txn.contains(&cf, b"key").unwrap()); +} + +#[test] +fn test_empty_value_is_distinct_from_absent() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + put_all(&db, &cf, &[(b"empty", b"")]); + + let txn = db.begin_transaction().unwrap(); + assert_eq!(txn.get(&cf, b"empty").unwrap(), Vec::::new()); + assert!(txn.contains(&cf, b"empty").unwrap()); + + let mut iter = txn.new_iterator(&cf).unwrap(); + iter.seek_to_first().unwrap(); + assert!(iter.is_valid()); + assert_eq!(iter.key_value().unwrap(), (b"empty".to_vec(), Vec::new())); + assert_eq!(iter.value().unwrap(), Vec::::new()); +} + +#[test] +fn test_empty_key_is_rejected() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + let txn = db.begin_transaction().unwrap(); + assert!(txn.put(&cf, b"", b"v").is_err()); +} + +#[test] +fn test_large_value_roundtrip() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + let big = vec![0xAB_u8; 1 << 20]; + put_all(&db, &cf, &[(b"big", &big)]); + db.flush_memtable().unwrap(); + assert_eq!( + db.begin_transaction().unwrap().get(&cf, b"big").unwrap(), + big + ); +} + +#[test] +fn test_get_notrack_and_contains() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + put_all(&db, &cf, &[(b"k", b"v")]); + + let txn = db.begin_transaction().unwrap(); + assert_eq!(txn.get_notrack(&cf, b"k").unwrap(), b"v"); + assert!(txn.contains(&cf, b"k").unwrap()); + assert!(!txn.contains(&cf, b"missing").unwrap()); + assert!(txn.get_notrack(&cf, b"missing").unwrap_err().is_not_found()); +} + +#[test] +fn test_multi_operation_transaction() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + put_all( + &db, + &cf, + &[ + (b"key1", b"value1"), + (b"key2", b"value2"), + (b"key3", b"value3"), + ], + ); + + let txn = db.begin_transaction().unwrap(); + for i in 1..=3 { + let value = txn.get(&cf, format!("key{i}").as_bytes()).unwrap(); + assert_eq!(value, format!("value{i}").as_bytes()); + } +} + +#[test] +fn test_read_your_own_writes() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + let txn = db.begin_transaction().unwrap(); + txn.put(&cf, b"k", b"pending").unwrap(); + assert_eq!(txn.get(&cf, b"k").unwrap(), b"pending"); +} + +#[test] +fn test_multi_column_family_transaction() { + let (db, _dir) = create_test_db(); + let users = create_cf(&db, "users"); + let orders = create_cf(&db, "orders"); + + let mut txn = db.begin_transaction().unwrap(); + txn.put(&users, b"u1", b"alice").unwrap(); + txn.put(&orders, b"o1", b"u1").unwrap(); + txn.commit().unwrap(); + + let txn = db.begin_transaction().unwrap(); + assert_eq!(txn.get(&users, b"u1").unwrap(), b"alice"); + assert_eq!(txn.get(&orders, b"o1").unwrap(), b"u1"); + assert!(txn.get(&users, b"o1").unwrap_err().is_not_found()); +} + +#[test] +fn test_rollback() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + + let mut txn = db.begin_transaction().unwrap(); + txn.put(&cf, b"k", b"v").unwrap(); + txn.rollback().unwrap(); + drop(txn); + + assert!( + db.begin_transaction() + .unwrap() + .get(&cf, b"k") + .unwrap_err() + .is_not_found() + ); +} + +#[test] +fn test_drop_uncommitted_transaction_discards_writes() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + { + let txn = db.begin_transaction().unwrap(); + txn.put(&cf, b"k", b"v").unwrap(); + } + assert!( + db.begin_transaction() + .unwrap() + .get(&cf, b"k") + .unwrap_err() + .is_not_found() + ); +} + +#[test] +fn test_savepoints() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + + let mut txn = db.begin_transaction().unwrap(); + txn.put(&cf, b"k1", b"v1").unwrap(); + txn.savepoint("sp1").unwrap(); + txn.put(&cf, b"k2", b"v2").unwrap(); + txn.rollback_to_savepoint("sp1").unwrap(); + txn.savepoint("sp2").unwrap(); + txn.release_savepoint("sp2").unwrap(); + assert!( + txn.rollback_to_savepoint("nope") + .unwrap_err() + .is_not_found() + ); + txn.commit().unwrap(); + + let txn = db.begin_transaction().unwrap(); + assert_eq!(txn.get(&cf, b"k1").unwrap(), b"v1"); + assert!(txn.get(&cf, b"k2").unwrap_err().is_not_found()); +} + +#[test] +fn test_transaction_reset() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + + let mut txn = db.begin_transaction().unwrap(); + txn.put(&cf, b"a", b"1").unwrap(); + txn.commit().unwrap(); + + txn.reset(IsolationLevel::Serializable).unwrap(); + txn.put(&cf, b"b", b"2").unwrap(); + txn.commit().unwrap(); + + txn.reset(IsolationLevel::ReadCommitted).unwrap(); + txn.put(&cf, b"c", b"3").unwrap(); + txn.rollback().unwrap(); + txn.reset(IsolationLevel::ReadCommitted).unwrap(); + + assert_eq!(txn.get(&cf, b"a").unwrap(), b"1"); + assert_eq!(txn.get(&cf, b"b").unwrap(), b"2"); + assert!(txn.get(&cf, b"c").unwrap_err().is_not_found()); +} + +#[test] +fn test_single_delete() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + put_all(&db, &cf, &[(b"once", b"v"), (b"other", b"w")]); + + let mut txn = db.begin_transaction().unwrap(); + txn.single_delete(&cf, b"once").unwrap(); + txn.commit().unwrap(); + db.flush_memtable().unwrap(); + + let txn = db.begin_transaction().unwrap(); + assert!(txn.get(&cf, b"once").unwrap_err().is_not_found()); + assert_eq!(txn.get(&cf, b"other").unwrap(), b"w"); +} + +// ===== range and prefix deletes ===== + +#[test] +fn test_delete_range() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + put_all( + &db, + &cf, + &[(b"a", b"1"), (b"b", b"2"), (b"c", b"3"), (b"d", b"4")], + ); + + let mut txn = db.begin_transaction().unwrap(); + txn.delete_range(&cf, b"b", Some(b"d")).unwrap(); + txn.commit().unwrap(); + + let txn = db.begin_transaction().unwrap(); + assert_eq!(collect_keys(&txn, &cf), vec![b"a".to_vec(), b"d".to_vec()]); +} + +#[test] +fn test_delete_range_open_ended_and_survived_by_newer_write() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + put_all(&db, &cf, &[(b"a", b"1"), (b"m", b"2"), (b"z", b"3")]); + db.flush_memtable().unwrap(); + + let mut txn = db.begin_transaction().unwrap(); + txn.delete_range(&cf, b"b", None).unwrap(); + txn.commit().unwrap(); + + // a write committed after the range delete survives it + put_all(&db, &cf, &[(b"q", b"new")]); + + let txn = db.begin_transaction().unwrap(); + assert_eq!(collect_keys(&txn, &cf), vec![b"a".to_vec(), b"q".to_vec()]); +} + +#[test] +fn test_delete_prefix() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + put_all( + &db, + &cf, + &[(b"user:1", b"a"), (b"user:2", b"b"), (b"order:1", b"c")], + ); + + let mut txn = db.begin_transaction().unwrap(); + txn.delete_prefix(&cf, b"user:").unwrap(); + txn.commit().unwrap(); + + let txn = db.begin_transaction().unwrap(); + assert_eq!(collect_keys(&txn, &cf), vec![b"order:1".to_vec()]); +} + +#[test] +fn test_range_delete_bound_validation() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + let txn = db.begin_transaction().unwrap(); + assert!(txn.delete_range(&cf, b"", None).is_err()); + assert!(txn.delete_prefix(&cf, b"").is_err()); + assert!(txn.delete_prefix(&cf, &[b'x'; 257]).is_err()); + txn.delete_prefix(&cf, &[b'x'; 256]).unwrap(); +} + +// ===== TTL ===== + +#[test] +fn test_put_with_ttl_is_relative() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + + let mut txn = db.begin_transaction().unwrap(); + txn.put_with_ttl(&cf, b"temp", b"v", Duration::from_secs(60)) + .unwrap(); + txn.put(&cf, b"forever", b"v").unwrap(); + txn.commit().unwrap(); + + let txn = db.begin_transaction().unwrap(); + let expires = txn.get_expiry(&cf, b"temp").unwrap().unwrap(); + let now = now_secs(); + assert!( + expires >= now + 58 && expires <= now + 61, + "expires={expires} now={now}" + ); + assert_eq!(txn.get_expiry(&cf, b"forever").unwrap(), None); + assert!(txn.get_expiry(&cf, b"missing").unwrap_err().is_not_found()); +} + +#[test] +fn test_ttl_expires() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + + let mut txn = db.begin_transaction().unwrap(); + txn.put_with_ttl(&cf, b"short", b"v", Duration::from_millis(1)) + .unwrap(); + txn.commit().unwrap(); + + // sub-second lifetimes round up to one second + assert_eq!( + db.begin_transaction().unwrap().get(&cf, b"short").unwrap(), + b"v" + ); + std::thread::sleep(Duration::from_millis(2100)); + assert!( + db.begin_transaction() + .unwrap() + .get(&cf, b"short") + .unwrap_err() + .is_not_found() + ); +} + +#[test] +fn test_zero_ttl_rejected() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + let txn = db.begin_transaction().unwrap(); + let err = txn + .put_with_ttl(&cf, b"k", b"v", Duration::ZERO) + .unwrap_err(); + assert_eq!(err.code(), Some(ErrorCode::InvalidArgs)); +} + +#[test] +fn test_iterator_ttl() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + let mut txn = db.begin_transaction().unwrap(); + txn.put_with_ttl(&cf, b"a", b"1", Duration::from_secs(100)) + .unwrap(); + txn.put(&cf, b"b", b"2").unwrap(); + txn.commit().unwrap(); + + let txn = db.begin_transaction().unwrap(); + let mut iter = txn.new_iterator(&cf).unwrap(); + iter.seek_to_first().unwrap(); + assert!(iter.expires_at().unwrap().unwrap() > now_secs()); + iter.next().unwrap(); + assert_eq!(iter.expires_at().unwrap(), None); +} + +// ===== iterators ===== + +#[test] +fn test_iterator_forward_backward() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + put_all(&db, &cf, &[(b"a", b"1"), (b"b", b"2"), (b"c", b"3")]); + + let txn = db.begin_transaction().unwrap(); + assert_eq!( + collect_keys(&txn, &cf), + vec![b"a".to_vec(), b"b".to_vec(), b"c".to_vec()] + ); + + let mut iter = txn.new_iterator(&cf).unwrap(); + iter.seek_to_last().unwrap(); + let mut keys = Vec::new(); + while iter.is_valid() { + keys.push(iter.key().unwrap()); + iter.prev().unwrap(); + } + assert_eq!(keys, vec![b"c".to_vec(), b"b".to_vec(), b"a".to_vec()]); +} + +#[test] +fn test_iterator_seek() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + put_all(&db, &cf, &[(b"b", b"1"), (b"d", b"2"), (b"f", b"3")]); + + let txn = db.begin_transaction().unwrap(); + let mut iter = txn.new_iterator(&cf).unwrap(); + + iter.seek(b"c").unwrap(); + assert_eq!(iter.key().unwrap(), b"d"); + iter.seek_for_prev(b"e").unwrap(); + assert_eq!(iter.key().unwrap(), b"d"); + iter.seek(b"z").unwrap(); + assert!(!iter.is_valid()); + iter.seek_for_prev(b"a").unwrap(); + assert!(!iter.is_valid()); + assert!(iter.key().unwrap_err().is_not_found()); +} + +#[test] +fn test_iterator_empty_family() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + let txn = db.begin_transaction().unwrap(); + assert!(collect_keys(&txn, &cf).is_empty()); +} + +#[test] +fn test_iterator_over_flushed_and_memtable_data() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + put_all(&db, &cf, &[(b"a", b"1"), (b"c", b"3")]); + db.flush_memtable().unwrap(); + put_all(&db, &cf, &[(b"b", b"2"), (b"c", b"33")]); + + let txn = db.begin_transaction().unwrap(); + let mut iter = txn.new_iterator(&cf).unwrap(); + iter.seek_to_first().unwrap(); + let mut pairs = Vec::new(); + while iter.is_valid() { + pairs.push(iter.key_value().unwrap()); + iter.next().unwrap(); + } + assert_eq!( + pairs, + vec![ + (b"a".to_vec(), b"1".to_vec()), + (b"b".to_vec(), b"2".to_vec()), + (b"c".to_vec(), b"33".to_vec()), + ] + ); +} + +#[test] +fn test_range_iterator() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + let pairs: Vec<(Vec, Vec)> = (0..50) + .map(|i| (format!("k{i:03}").into_bytes(), b"v".to_vec())) + .collect(); + let refs: Vec<(&[u8], &[u8])> = pairs.iter().map(|(k, v)| (&k[..], &v[..])).collect(); + put_all(&db, &cf, &refs); + db.flush_memtable().unwrap(); + + let txn = db.begin_transaction().unwrap(); + let mut iter = txn.new_range_iterator(&cf, b"k010", b"k019").unwrap(); + iter.seek(b"k010").unwrap(); + let mut keys = Vec::new(); + while iter.is_valid() { + let k = iter.key().unwrap(); + if k.as_slice() > b"k019".as_slice() { + break; + } + keys.push(k); + iter.next().unwrap(); + } + assert_eq!(keys.len(), 10); + assert_eq!(keys[0], b"k010"); + assert_eq!(keys[9], b"k019"); +} + +#[test] +fn test_iterator_does_not_leak_on_many_reads() { + // Exercises the key/value buffer frees; under a leak checker this would report. + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + put_all(&db, &cf, &[(b"k", b"v")]); + let txn = db.begin_transaction().unwrap(); + let mut iter = txn.new_iterator(&cf).unwrap(); + iter.seek_to_first().unwrap(); + for _ in 0..10_000 { + let _ = iter.key().unwrap(); + let _ = iter.value().unwrap(); + let _ = iter.key_value().unwrap(); + } +} + +// ===== isolation, snapshots, point-in-time reads ===== + +#[test] +fn test_isolation_levels_begin() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + for level in [ + IsolationLevel::ReadUncommitted, + IsolationLevel::ReadCommitted, + IsolationLevel::RepeatableRead, + IsolationLevel::Snapshot, + IsolationLevel::Serializable, + ] { + let mut txn = db.begin_transaction_with_isolation(level).unwrap(); + txn.put(&cf, b"k", b"v").unwrap(); + txn.commit().unwrap(); + } +} + +#[test] +fn test_repeatable_read_does_not_see_later_commits() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + put_all(&db, &cf, &[(b"k", b"old")]); + + let reader = db + .begin_transaction_with_isolation(IsolationLevel::RepeatableRead) + .unwrap(); + assert_eq!(reader.get(&cf, b"k").unwrap(), b"old"); + + put_all(&db, &cf, &[(b"k", b"new")]); + assert_eq!(reader.get(&cf, b"k").unwrap(), b"old"); + + let rc = db.begin_transaction().unwrap(); + assert_eq!(rc.get(&cf, b"k").unwrap(), b"new"); +} + +#[test] +fn test_snapshot_isolation_write_conflict() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + put_all(&db, &cf, &[(b"k", b"0")]); + + let mut t1 = db + .begin_transaction_with_isolation(IsolationLevel::Snapshot) + .unwrap(); + let mut t2 = db + .begin_transaction_with_isolation(IsolationLevel::Snapshot) + .unwrap(); + t1.put(&cf, b"k", b"1").unwrap(); + t2.put(&cf, b"k", b"2").unwrap(); + t1.commit().unwrap(); + let err = t2.commit().unwrap_err(); + assert!(err.is_conflict(), "{err}"); + + let stats = db.get_db_stats().unwrap(); + assert!(stats.txn_conflicts >= 1); +} + +#[test] +fn test_begin_transaction_cf_uses_family_isolation() { + let (db, _dir) = create_test_db(); + db.create_column_family( + "rr", + ColumnFamilyConfig::default().default_isolation_level(IsolationLevel::RepeatableRead), + ) + .unwrap(); + let cf = db.get_column_family("rr").unwrap(); + put_all(&db, &cf, &[(b"k", b"old")]); + + let txn = db.begin_transaction_cf(&cf).unwrap(); + assert_eq!(txn.get(&cf, b"k").unwrap(), b"old"); + put_all(&db, &cf, &[(b"k", b"new")]); + assert_eq!(txn.get(&cf, b"k").unwrap(), b"old"); +} + +#[test] +fn test_snapshot_point_in_time_read() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + put_all(&db, &cf, &[(b"k", b"v1")]); + + let snap = db.snapshot().unwrap(); + assert!(snap.seq() > 0); + + put_all(&db, &cf, &[(b"k", b"v2"), (b"other", b"x")]); + db.flush_memtable().unwrap(); + + let old = db.begin_transaction_at_snapshot(&snap).unwrap(); + assert_eq!(old.get(&cf, b"k").unwrap(), b"v1"); + assert!(old.get(&cf, b"other").unwrap_err().is_not_found()); + + let now = db.begin_transaction().unwrap(); + assert_eq!(now.get(&cf, b"k").unwrap(), b"v2"); +} + +#[test] +fn test_snapshot_shared_across_threads() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + put_all(&db, &cf, &[(b"k", b"v1")]); + let snap = db.snapshot().unwrap(); + put_all(&db, &cf, &[(b"k", b"v2")]); + + std::thread::scope(|s| { + for _ in 0..4 { + s.spawn(|| { + let txn = db.begin_transaction_at_snapshot(&snap).unwrap(); + assert_eq!(txn.get(&cf, b"k").unwrap(), b"v1"); + }); + } + }); +} + +#[test] +fn test_begin_at_seq_and_read_snapshot() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + put_all(&db, &cf, &[(b"k", b"v1")]); + + // hold the floor with a repeatable read transaction and remember its sequence + let holder = db + .begin_transaction_with_isolation(IsolationLevel::RepeatableRead) + .unwrap(); + let seq = holder.read_snapshot(); + assert!(seq > 0 && seq != u64::MAX); + assert!(db.oldest_readable_seq() <= seq); + + put_all(&db, &cf, &[(b"k", b"v2")]); + + let at = db.begin_transaction_at_seq(seq).unwrap(); + assert_eq!(at.get(&cf, b"k").unwrap(), b"v1"); + + let ru = db + .begin_transaction_with_isolation(IsolationLevel::ReadUncommitted) + .unwrap(); + assert_eq!(ru.read_snapshot(), u64::MAX); +} + +// ===== transaction lifecycle, timeouts, aborts ===== + +#[test] +fn test_transaction_state() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + let mut txn = db.begin_transaction().unwrap(); + assert_eq!(txn.state().unwrap(), TxnState::Active); + txn.put(&cf, b"k", b"v").unwrap(); + txn.commit().unwrap(); + assert_eq!(txn.state().unwrap(), TxnState::Committed); + + let mut txn = db.begin_transaction().unwrap(); + txn.put(&cf, b"k", b"v").unwrap(); + txn.rollback().unwrap(); + assert_eq!(txn.state().unwrap(), TxnState::Aborted); +} + +#[test] +fn test_transaction_timeout() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + let mut txn = db.begin_transaction().unwrap(); + txn.set_timeout(Some(Duration::from_secs(1))).unwrap(); + txn.put(&cf, b"k", b"v").unwrap(); + std::thread::sleep(Duration::from_millis(3200)); + let err = txn.put(&cf, b"k2", b"v").unwrap_err(); + assert_eq!(err.code(), Some(ErrorCode::TxnExpired)); + + let mut txn = db.begin_transaction().unwrap(); + txn.set_timeout(Some(Duration::from_secs(1))).unwrap(); + txn.set_timeout(None).unwrap(); + std::thread::sleep(Duration::from_millis(3200)); + txn.put(&cf, b"k", b"v").unwrap(); + txn.commit().unwrap(); +} + +#[test] +fn test_db_level_txn_timeout() { + let dir = TempDir::new().unwrap(); + let db = TidesDB::open(test_config(&dir).txn_timeout_seconds(1)).unwrap(); + let cf = create_cf(&db, "cf"); + let mut txn = db.begin_transaction().unwrap(); + std::thread::sleep(Duration::from_millis(3200)); + // the first operation past the deadline expires the transaction + assert_eq!( + txn.put(&cf, b"k", b"v").unwrap_err().code(), + Some(ErrorCode::TxnExpired) + ); + assert!(txn.commit().is_err()); +} + +#[test] +fn test_abort_handle_from_other_thread() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + let mut txn = db.begin_transaction().unwrap(); + txn.put(&cf, b"k", b"v").unwrap(); + + let handle = txn.abort_handle(); + std::thread::scope(|s| { + s.spawn(move || handle.request_abort()); + }); + + let err = txn.commit().unwrap_err(); + assert_eq!(err.code(), Some(ErrorCode::TxnAborted)); + drop(txn); + assert!( + db.begin_transaction() + .unwrap() + .get(&cf, b"k") + .unwrap_err() + .is_not_found() + ); +} + +#[test] +fn test_abort_handle_outliving_transaction() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + let handle; + { + let txn = db.begin_transaction().unwrap(); + txn.put(&cf, b"k", b"v").unwrap(); + handle = txn.abort_handle(); + } + // the transaction was rolled back on drop; aborting it now is a no-op + handle.request_abort(); + drop(handle); + assert!( + db.begin_transaction() + .unwrap() + .get(&cf, b"k") + .unwrap_err() + .is_not_found() + ); +} + +// ===== two-phase commit ===== + +#[test] +fn test_two_phase_commit() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + + let mut txn = db.begin_transaction().unwrap(); + txn.put(&cf, b"k", b"v").unwrap(); + txn.prepare(b"xid-1").unwrap(); + assert_eq!(txn.state().unwrap(), TxnState::Prepared); + + // prepared writes are invisible + assert!( + db.begin_transaction() + .unwrap() + .get(&cf, b"k") + .unwrap_err() + .is_not_found() + ); + + txn.commit_prepared().unwrap(); + assert_eq!(txn.state().unwrap(), TxnState::Committed); + assert_eq!( + db.begin_transaction().unwrap().get(&cf, b"k").unwrap(), + b"v" + ); +} + +#[test] +fn test_two_phase_rollback() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + let mut txn = db.begin_transaction().unwrap(); + txn.put(&cf, b"k", b"v").unwrap(); + txn.prepare(b"xid-2").unwrap(); + txn.rollback_prepared().unwrap(); + drop(txn); + assert!( + db.begin_transaction() + .unwrap() + .get(&cf, b"k") + .unwrap_err() + .is_not_found() + ); +} + +#[test] +fn test_commit_prepared_requires_prepare() { + let (db, _dir) = create_test_db(); + let mut txn = db.begin_transaction().unwrap(); + assert_eq!( + txn.commit_prepared().unwrap_err().code(), + Some(ErrorCode::InvalidArgs) + ); +} + +#[test] +fn test_recover_prepared_after_restart() { + let dir = TempDir::new().unwrap(); + { + let db = TidesDB::open(test_config(&dir)).unwrap(); + let cf = create_cf(&db, "cf"); + let mut txn = db.begin_transaction().unwrap(); + txn.put(&cf, b"k", b"v").unwrap(); + txn.prepare(b"xid-recover").unwrap(); + // the coordinator goes away: dropping a prepared transaction leaves it in doubt on disk + drop(txn); + } + + let db = TidesDB::open(test_config(&dir)).unwrap(); + let cf = db.get_column_family("cf").unwrap(); + assert!( + db.begin_transaction() + .unwrap() + .get(&cf, b"k") + .unwrap_err() + .is_not_found() + ); + + let mut recovered = db.recover_prepared().unwrap(); + assert_eq!(recovered.len(), 1); + assert_eq!(recovered[0].xid, b"xid-recover"); + assert_eq!(recovered[0].txn.state().unwrap(), TxnState::Prepared); + recovered[0].txn.commit_prepared().unwrap(); + drop(recovered); + + assert_eq!( + db.begin_transaction().unwrap().get(&cf, b"k").unwrap(), + b"v" + ); +} + +// ===== commit hooks ===== + +#[test] +fn test_commit_hook_puts_and_deletes() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + type Batches = Arc, u64)>>>; + let seen: Batches = Arc::default(); + let sink = seen.clone(); + cf.set_commit_hook(move |ops, seq| { + sink.lock().unwrap().push((ops.to_vec(), seq)); + 0 + }) + .unwrap(); + + let mut txn = db.begin_transaction().unwrap(); + txn.put(&cf, b"a", b"1").unwrap(); + txn.put_with_ttl(&cf, b"b", b"", Duration::from_secs(100)) + .unwrap(); + txn.commit().unwrap(); + + let mut txn = db.begin_transaction().unwrap(); + txn.delete(&cf, b"a").unwrap(); + txn.commit().unwrap(); + + let seen = seen.lock().unwrap(); + assert_eq!(seen.len(), 2); + let (batch, seq1) = &seen[0]; + assert_eq!(batch.len(), 2); + let a = batch.iter().find(|op| op.key == b"a").unwrap(); + assert_eq!(a.value.as_deref(), Some(&b"1"[..])); + assert_eq!(a.expires_at, None); + let b = batch.iter().find(|op| op.key == b"b").unwrap(); + assert_eq!(b.value.as_deref(), Some(&b""[..])); + assert!(b.expires_at.unwrap() > now_secs()); + + let (batch, seq2) = &seen[1]; + assert!(batch[0].is_delete && batch[0].value.is_none()); + assert_eq!(batch[0].range_end, None); + assert!(seq2 > seq1); + assert_eq!(a.range_end, None); +} + +#[test] +fn test_commit_hook_range_and_prefix_deletes() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + let seen: Arc>> = Arc::default(); + let sink = seen.clone(); + cf.set_commit_hook(move |ops, _| { + sink.lock().unwrap().extend_from_slice(ops); + 0 + }) + .unwrap(); + + let commit = |f: &dyn Fn(&Transaction<'_>) -> Result<()>| { + let mut txn = db.begin_transaction().unwrap(); + f(&txn).unwrap(); + txn.commit().unwrap(); + }; + commit(&|t| t.delete_range(&cf, b"b", Some(b"d"))); + commit(&|t| t.delete_prefix(&cf, b"user:")); + commit(&|t| t.delete_prefix(&cf, b"ab\xff\xff")); + commit(&|t| t.delete_range(&cf, b"x", None)); + commit(&|t| t.delete_prefix(&cf, b"\xff")); + commit(&|t| t.single_delete(&cf, b"s")); + + let seen = seen.lock().unwrap(); + let ops: Vec<_> = seen + .iter() + .map(|op| { + assert!(op.is_delete); + assert_eq!(op.value, None); + (op.key.as_slice(), op.range_end.as_deref()) + }) + .collect(); + assert_eq!( + ops, + vec![ + (&b"b"[..], Some(&b"d"[..])), + (&b"user:"[..], Some(&b"user;"[..])), + (&b"ab\xff\xff"[..], Some(&b"ac"[..])), + // unbounded above: indistinguishable from a point delete, as documented + (&b"x"[..], None), + (&b"\xff"[..], None), + (&b"s"[..], None), + ] + ); +} + +#[test] +fn test_delete_range_rejects_empty_upper_bound() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + let mut txn = db.begin_transaction().unwrap(); + txn.put(&cf, b"m", b"v").unwrap(); + txn.commit().unwrap(); + + let mut txn = db.begin_transaction().unwrap(); + let err = txn.delete_range(&cf, b"a", Some(b"")).unwrap_err(); + assert_eq!(err.code(), Some(ErrorCode::InvalidArgs)); + assert_eq!( + txn.delete_range(&cf, b"z", Some(b"a")).unwrap_err().code(), + Some(ErrorCode::InvalidArgs) + ); + txn.commit().unwrap(); + assert!(db.begin_transaction().unwrap().contains(&cf, b"m").unwrap()); +} + +#[test] +fn test_commit_hook_replace_and_clear() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + let first = Arc::new(AtomicUsize::new(0)); + let second = Arc::new(AtomicUsize::new(0)); + + let f = first.clone(); + cf.set_commit_hook(move |_, _| { + f.fetch_add(1, Ordering::SeqCst); + 0 + }) + .unwrap(); + put_all(&db, &cf, &[(b"k", b"1")]); + + let s = second.clone(); + cf.set_commit_hook(move |_, _| { + s.fetch_add(1, Ordering::SeqCst); + 0 + }) + .unwrap(); + put_all(&db, &cf, &[(b"k", b"2")]); + + cf.clear_commit_hook().unwrap(); + put_all(&db, &cf, &[(b"k", b"3")]); + + assert_eq!(first.load(Ordering::SeqCst), 1); + assert_eq!(second.load(Ordering::SeqCst), 1); +} + +#[test] +fn test_commit_hook_outlives_handle() { + let (db, _dir) = create_test_db(); + let count = Arc::new(AtomicUsize::new(0)); + { + let cf = create_cf(&db, "cf"); + let c = count.clone(); + cf.set_commit_hook(move |_, _| { + c.fetch_add(1, Ordering::SeqCst); + 0 + }) + .unwrap(); + } + // a fresh handle to the same family: the hook is still installed + let cf = db.get_column_family("cf").unwrap(); + put_all(&db, &cf, &[(b"k", b"v")]); + assert_eq!(count.load(Ordering::SeqCst), 1); +} + +#[test] +fn test_commit_hook_panic_does_not_abort_commit() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + cf.set_commit_hook(|_, _| panic!("hook failure")).unwrap(); + put_all(&db, &cf, &[(b"k", b"v")]); + assert_eq!( + db.begin_transaction().unwrap().get(&cf, b"k").unwrap(), + b"v" + ); +} + +#[test] +fn test_commit_hook_concurrent_commits() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + let total = Arc::new(AtomicU64::new(0)); + let t = total.clone(); + cf.set_commit_hook(move |ops, _| { + t.fetch_add(ops.len() as u64, Ordering::SeqCst); + 0 + }) + .unwrap(); + + std::thread::scope(|s| { + for thread in 0..4 { + let (db, cf) = (&db, &cf); + s.spawn(move || { + for i in 0..50 { + let mut txn = db.begin_transaction().unwrap(); + txn.put(cf, format!("t{thread}-{i}").as_bytes(), b"v") + .unwrap(); + txn.commit().unwrap(); + } + }); + } + }); + assert_eq!(total.load(Ordering::SeqCst), 200); +} + +// ===== configuration ===== + +#[test] +fn test_default_config_tracks_library() { + let config = Config::default(); + assert!(config.db_path.is_empty()); + assert!(config.num_flush_threads > 0); + assert_eq!(config.txn_timeout_seconds, 0); + + let cf = ColumnFamilyConfig::default(); + assert!(cf.level_size_ratio > 0); + assert!(cf.enable_bloom_filter); + assert!(cf.encoding_pipeline.is_empty()); + assert_eq!(cf.default_isolation_level, IsolationLevel::ReadCommitted); +} + +#[test] +fn test_full_db_config_opens() { + let dir = TempDir::new().unwrap(); + let config = Config::new(dir.path()) + .num_flush_threads(1) + .num_compaction_threads(1) + .log_level(LogLevel::None) + .log_to_file(true) + .log_truncation_at(1 << 20) + .memtable_write_buffer_size(4 << 20) + .memtable_skip_list_max_level(10) + .memtable_skip_list_probability(0.25) + .memtable_sync_mode(SyncMode::Interval) + .memtable_sync_interval_us(10_000) + .value_separation_threshold(1024) + .vlog_segment_size(64 << 20) + .memtable_l0_queue_stall_threshold(16) + .memtable_idle_flush_seconds(30) + .txn_timeout_seconds(0); + let db = TidesDB::open(config).unwrap(); + let cf = create_cf(&db, "cf"); + put_all(&db, &cf, &[(b"k", b"v")]); + db.sync_wal().unwrap(); +} + +#[test] +fn test_custom_cf_config_roundtrip_through_stats() { + let (db, _dir) = create_test_db(); + let config = ColumnFamilyConfig::new() + .level_size_ratio(8) + .min_levels(3) + .dividing_level_offset(1) + .keep_values_inline(true) + .btree_klog_block_size(8192) + .enable_bloom_filter(true) + .bloom_fpr(0.001) + .default_isolation_level(IsolationLevel::Snapshot) + .l1_file_count_trigger(6) + .tombstone_density_trigger(0.4) + .tombstone_density_min_entries(512); + db.create_column_family("custom", config.clone()).unwrap(); + let cf = db.get_column_family("custom").unwrap(); + assert_eq!(cf.get_stats().unwrap().config, config); +} + +#[test] +fn test_compression_pipeline() { + let (db, _dir) = create_test_db(); + assert!(CompressionAlgorithm::None.is_available()); + + let available: Vec<_> = [ + CompressionAlgorithm::Snappy, + CompressionAlgorithm::Lz4, + CompressionAlgorithm::Zstd, + CompressionAlgorithm::Lz4Fast, + ] + .into_iter() + .filter(|a| a.is_available()) + .collect(); + // the features only choose the codecs of the bundled build; a system TidesDB has its own + if cfg!(not(tidesdb_system)) { + assert_eq!( + CompressionAlgorithm::Lz4.is_available(), + cfg!(feature = "lz4") + ); + assert_eq!( + CompressionAlgorithm::Zstd.is_available(), + cfg!(feature = "zstd") + ); + assert_eq!( + CompressionAlgorithm::Snappy.is_available(), + cfg!(feature = "snappy") + ); + } + + for algo in available { + let name = format!("{algo:?}"); + let config = ColumnFamilyConfig::new().compression_algorithm(algo); + db.create_column_family(&name, config).unwrap(); + let cf = db.get_column_family(&name).unwrap(); + assert_eq!(cf.get_stats().unwrap().config.encoding_pipeline, vec![algo]); + + let value = vec![b'x'; 64 * 1024]; + put_all(&db, &cf, &[(b"k", &value)]); + db.flush_memtable().unwrap(); + assert_eq!( + db.begin_transaction().unwrap().get(&cf, b"k").unwrap(), + value + ); + } + + let none = ColumnFamilyConfig::new().compression_algorithm(CompressionAlgorithm::None); + assert!(none.encoding_pipeline.is_empty()); +} + +#[test] +fn test_encoding_pipeline_too_long() { + let (db, _dir) = create_test_db(); + let config = ColumnFamilyConfig::new().encoding_pipeline(vec![CompressionAlgorithm::Lz4; 9]); + let err = db.create_column_family("long", config).unwrap_err(); + assert_eq!(err.code(), Some(ErrorCode::InvalidArgs)); +} + +#[test] +fn test_update_runtime_config() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + let updated = ColumnFamilyConfig::default() + .l1_file_count_trigger(9) + .bloom_fpr(0.02); + cf.update_runtime_config(&updated, true).unwrap(); + let stats = cf.get_stats().unwrap(); + assert_eq!(stats.config.l1_file_count_trigger, 9); + assert_eq!(stats.config.bloom_fpr, 0.02); +} + +#[test] +fn test_raise_open_file_limit() { + let current = raise_open_file_limit(0); + assert!(current > 0); + assert!(raise_open_file_limit(current) >= current); +} + +// ===== maintenance ===== + +#[test] +fn test_flush_compact_and_status() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + for batch in 0..4 { + let mut txn = db.begin_transaction().unwrap(); + for i in 0..100 { + txn.put(&cf, format!("key{batch}-{i:03}").as_bytes(), b"value") + .unwrap(); + } + txn.commit().unwrap(); + db.flush_memtable().unwrap(); + } + let _ = db.is_flushing(); + let _ = cf.is_compacting(); + + match cf.compact() { + Ok(()) => {} + Err(e) if e.is_locked() => {} + Err(e) => panic!("compact failed: {e}"), + } + + let txn = db.begin_transaction().unwrap(); + assert_eq!(txn.get(&cf, b"key2-050").unwrap(), b"value"); + assert_eq!(collect_keys(&txn, &cf).len(), 400); +} + +#[test] +fn test_compact_range() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + put_all(&db, &cf, &[(b"a", b"1"), (b"m", b"2"), (b"z", b"3")]); + db.flush_memtable().unwrap(); + + match cf.compact_range(Some(b"a"), Some(b"n")) { + Ok(()) => {} + Err(e) if e.is_locked() => {} + Err(e) => panic!("compact_range failed: {e}"), + } + cf.compact_range(Some(b"m"), None).ok(); + assert_eq!( + cf.compact_range(None, None).unwrap_err().code(), + Some(ErrorCode::InvalidArgs) + ); + assert_eq!( + db.begin_transaction().unwrap().get(&cf, b"m").unwrap(), + b"2" + ); +} + +#[test] +fn test_sync_wal_and_checkpoint() { + let dir = TempDir::new().unwrap(); + let db = TidesDB::open(test_config(&dir).memtable_sync_mode(SyncMode::None)).unwrap(); + let cf = create_cf(&db, "cf"); + put_all(&db, &cf, &[(b"k", b"v")]); + db.sync_wal().unwrap(); + db.checkpoint().unwrap(); + assert!(cf.get_stats().unwrap().total_keys >= 1); +} + +#[test] +fn test_backup_is_openable() { + let (db, dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + put_all(&db, &cf, &[(b"k", b"v")]); + + let backup_dir = dir.path().join("backup"); + db.backup(&backup_dir).unwrap(); + + // a non-empty destination is refused + assert!(db.backup(&backup_dir).is_err()); + + let restored = TidesDB::open(Config::new(&backup_dir).log_level(LogLevel::Warn)).unwrap(); + let cf = restored.get_column_family("cf").unwrap(); + assert_eq!( + restored + .begin_transaction() + .unwrap() + .get(&cf, b"k") + .unwrap(), + b"v" + ); +} + +// ===== statistics ===== + +#[test] +fn test_cf_stats_after_flush() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + let mut txn = db.begin_transaction().unwrap(); + for i in 0..200 { + txn.put(&cf, format!("key{i:04}").as_bytes(), b"value") + .unwrap(); + } + txn.commit().unwrap(); + + let before = cf.get_stats().unwrap(); + assert!(before.unflushed_key_count > 0); + assert!(before.user_bytes_written > 0); + + db.flush_memtable().unwrap(); + let stats = cf.get_stats().unwrap(); + assert!(stats.num_levels >= 1); + assert_eq!(stats.level_sizes.len(), stats.num_levels as usize); + assert_eq!(stats.level_num_sstables.len(), stats.num_levels as usize); + assert_eq!(stats.level_key_counts.len(), stats.num_levels as usize); + assert_eq!( + stats.level_tombstone_counts.len(), + stats.num_levels as usize + ); + assert_eq!(stats.total_keys, 200); + assert!(stats.total_data_size > 0); + assert!(stats.avg_key_size > 0.0); + + // the flush counter is bumped just after the sstable is published, so it can lag the + // flush_memtable return by a moment + let deadline = std::time::Instant::now() + Duration::from_secs(5); + while cf.get_stats().unwrap().flush_bytes_written == 0 { + assert!( + std::time::Instant::now() < deadline, + "flush_bytes_written stayed 0" + ); + std::thread::sleep(Duration::from_millis(10)); + } +} + +#[test] +fn test_tombstone_stats_after_deletes() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + let mut txn = db.begin_transaction().unwrap(); + for i in 0..50 { + txn.put(&cf, format!("k{i:02}").as_bytes(), b"v").unwrap(); + } + txn.commit().unwrap(); + let mut txn = db.begin_transaction().unwrap(); + for i in 0..25 { + txn.delete(&cf, format!("k{i:02}").as_bytes()).unwrap(); + } + txn.commit().unwrap(); + db.flush_memtable().unwrap(); + + let stats = cf.get_stats().unwrap(); + assert!(stats.total_tombstones > 0); + assert!(stats.tombstone_ratio > 0.0); + assert!(stats.max_sst_density > 0.0); + assert!(stats.max_sst_density_level >= 1); +} + +#[test] +fn test_estimate_cardinality_and_range_stats() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + let mut txn = db.begin_transaction().unwrap(); + for i in 0..100 { + txn.put(&cf, format!("k{i:03}").as_bytes(), b"v").unwrap(); + } + txn.commit().unwrap(); + + // memtable-aware: counts unflushed keys too + let rs = cf.range_stats(b"k010", b"k020").unwrap(); + assert_eq!(rs.estimated_keys, 10); + assert!(rs.keys_exact); + + db.flush_memtable().unwrap(); + let estimate = cf.estimate_cardinality().unwrap(); + assert!((50..=200).contains(&estimate), "estimate={estimate}"); + let rs = cf.range_stats(b"k000", b"k100").unwrap(); + assert!(rs.sstables_overlapping >= 1); + assert!(rs.estimated_keys > 0); +} + +#[test] +fn test_db_stats() { + let (db, _dir) = create_test_db(); + let a = create_cf(&db, "a"); + let b = create_cf(&db, "b"); + put_all(&db, &a, &[(b"k", b"v")]); + put_all(&db, &b, &[(b"k", b"v")]); + + let stats = db.get_db_stats().unwrap(); + assert_eq!(stats.num_column_families, 2); + assert!(stats.global_seq > 0); + assert!(stats.txn_commits >= 2); + assert!(stats.memtable_bytes > 0); + assert!(stats.user_bytes_written > 0); + assert!(stats.wal_bytes_written > 0); + + db.flush_memtable().unwrap(); + let stats = db.get_db_stats().unwrap(); + assert!(stats.flush_count >= 1); + assert!(stats.total_sstable_count >= 1); +} + +#[test] +fn test_cache_stats() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + put_all(&db, &cf, &[(b"k", b"v")]); + db.flush_memtable().unwrap(); + for _ in 0..5 { + db.begin_transaction().unwrap().get(&cf, b"k").unwrap(); + } + let stats = db.get_cache_stats().unwrap(); + assert!(stats.enabled); + assert!(stats.num_partitions > 0); + assert!(stats.hits + stats.misses > 0); +} + +#[test] +fn test_stall_and_io_stats() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + put_all(&db, &cf, &[(b"k", b"v")]); + db.flush_memtable().unwrap(); + + let stalls = db.get_stall_stats().unwrap(); + let names: Vec<_> = stalls.iter().map(|(name, _)| name).collect(); + assert_eq!(names.len(), 5); + assert!( + names.iter().all(|n| !n.is_empty() && *n != "unknown"), + "{names:?}" + ); + + let io = db.get_io_stats().unwrap(); + let names: Vec<_> = io.iter().map(|(name, _)| name).collect(); + assert_eq!(names.len(), 3); + assert!(names.iter().all(|n| *n != "unknown"), "{names:?}"); + assert!(io.sstable.bytes > 0); +} + +#[test] +fn test_encoding_stats() { + let (db, _dir) = create_test_db(); + let cf = create_cf(&db, "cf"); + let value = vec![b'v'; 8192]; + put_all(&db, &cf, &[(b"k", &value)]); + db.flush_memtable().unwrap(); + + let klog = db.get_klog_encoding_stats().unwrap(); + assert!(!klog.is_empty()); + let verbatim = klog.iter().find(|e| e.ids.is_empty()).unwrap(); + assert!(verbatim.item_count >= 1); + assert!(verbatim.logical_bytes > 0); + + // just exercise the call; whether anything separated depends on the threshold + let _ = db.get_vlog_encoding_stats().unwrap(); +} + +// ===== errors ===== + +#[test] +fn test_error_codes_roundtrip() { + for code in -17..=-1 { + let ec = ErrorCode::from_code(code); + assert_eq!(ec.code(), code, "{ec:?}"); + assert!(!ec.to_string().is_empty()); + } + assert_eq!(ErrorCode::from_code(-99), ErrorCode::Unknown); + assert_eq!(ErrorCode::from_code(-14), ErrorCode::TxnExpired); + assert_eq!(ErrorCode::from_code(-15), ErrorCode::NoSpace); + assert_eq!(ErrorCode::from_code(-16), ErrorCode::TxnAborted); + assert_eq!(ErrorCode::from_code(-17), ErrorCode::TooOld); +} + +#[test] +fn test_error_display_uses_library_message() { + let err = Error::from_code(-3, "lookup"); + let text = err.to_string(); + assert!(text.starts_with("lookup: "), "{text}"); + assert!(text.len() > "lookup: ".len()); +} + +#[test] +fn test_linked_version_matches_major() { + // crate N.x binds TidesDB (N+9).x + let crate_major: u32 = env!("CARGO_PKG_VERSION_MAJOR").parse().unwrap(); + let major = |v: &str| v.split('.').next().unwrap().parse::().unwrap(); + assert_eq!(major(TIDESDB_VERSION), crate_major + 9); + assert_eq!(major(TIDESDB_LINKED_VERSION), major(TIDESDB_VERSION)); + if cfg!(not(tidesdb_system)) { + assert_eq!(TIDESDB_LINKED_VERSION, TIDESDB_VERSION); + } +} + +/// The allocator features must actually replace the process allocator. +#[cfg(target_os = "linux")] +#[test] +fn test_allocator_feature_is_linked() { + let maps = std::fs::read_to_string("/proc/self/maps").unwrap(); + for (enabled, name) in [ + (cfg!(feature = "jemalloc"), "libjemalloc"), + (cfg!(feature = "mimalloc"), "libmimalloc"), + (cfg!(feature = "tcmalloc"), "libtcmalloc"), + ] { + assert_eq!(maps.contains(name), enabled, "{name} mapped: {}", !enabled); + } +} diff --git a/src/transaction.rs b/src/transaction.rs index 9331d18..076f1fc 100644 --- a/src/transaction.rs +++ b/src/transaction.rs @@ -6,17 +6,56 @@ //! Transaction operations for TidesDB. use crate::config::IsolationLevel; -use crate::db::ColumnFamily; +use crate::db::{ColumnFamily, TidesDB, take_c_buffer, ttl_from_c}; use crate::error::{Error, Result, check_result}; use crate::ffi; use crate::iterator::Iterator; +use std::cell::Cell; use std::ffi::CString; +use std::marker::PhantomData; use std::ptr; +use std::sync::Arc; +use std::time::Duration; -/// A transaction in TidesDB. +/// Lifecycle state of a transaction, see [`Transaction::state`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TxnState { + /// Buffering writes, not yet resolved + Active, + /// Durably prepared under an xid, awaiting commit or rollback + Prepared, + /// Committed and applied + Committed, + /// Rolled back, aborted or expired + Aborted, +} + +/// Owns the C transaction. Shared with [`AbortHandle`]s so that `tidesdb_txn_request_abort` +/// can never run against a freed transaction. +struct TxnHandle { + txn: *mut ffi::tidesdb_txn_t, +} + +// Only `tidesdb_txn_request_abort` is reached through a shared handle, and that is the one +// call the C library allows from a thread other than the owner. Every other call goes +// through `Transaction`, which is not `Sync`. +unsafe impl Send for TxnHandle {} +unsafe impl Sync for TxnHandle {} + +impl Drop for TxnHandle { + fn drop(&mut self) { + // rolls back if still open + unsafe { ffi::tidesdb_txn_free(self.txn) }; + } +} + +/// A transaction. /// -/// Transactions provide atomic operations on the database. -/// The transaction is automatically freed when dropped. +/// All reads and writes go through a transaction. Writes are buffered until +/// [`commit`](Self::commit); dropping an uncommitted transaction rolls it back. +/// +/// A transaction may be moved to another thread but not shared between threads. To abort +/// it from another thread, use an [`AbortHandle`]. /// /// # Example /// @@ -28,263 +67,440 @@ use std::ptr; /// let cf = db.get_column_family("my_cf")?; /// /// let mut txn = db.begin_transaction()?; -/// txn.put(&cf, b"key", b"value", -1)?; +/// txn.put(&cf, b"key", b"value")?; /// txn.commit()?; /// # Ok::<(), tidesdb::Error>(()) /// ``` -pub struct Transaction { - txn: *mut ffi::tidesdb_txn_t, - committed: bool, +pub struct Transaction<'db> { + handle: Arc, + db: *mut ffi::tidesdb_t, + // borrows the database; Cell makes it !Sync while staying Send + _marker: PhantomData<(&'db TidesDB, Cell<()>)>, } -// Transaction uses internal locking for thread safety -unsafe impl Send for Transaction {} +unsafe impl Send for Transaction<'_> {} -impl Transaction { - /// Create a new transaction wrapper. - pub(crate) fn new(txn: *mut ffi::tidesdb_txn_t) -> Self { +fn key_ptr(key: &[u8]) -> *const u8 { + // a dangling-but-non-null pointer for an empty slice; sizes of zero are never read + key.as_ptr() +} + +impl<'db> Transaction<'db> { + /// Wrap a C transaction. + /// + /// # Safety + /// + /// `txn` must be a live transaction of `db`, owned by the caller. + pub(crate) unsafe fn from_raw(db: *mut ffi::tidesdb_t, txn: *mut ffi::tidesdb_txn_t) -> Self { Transaction { - txn, - committed: false, + handle: Arc::new(TxnHandle { txn }), + db, + _marker: PhantomData, } } - /// Adds a key-value pair to the transaction. + #[inline] + fn txn(&self) -> *mut ffi::tidesdb_txn_t { + self.handle.txn + } + + #[inline] + fn cf_ptr(&self, cf: &ColumnFamily<'_>) -> Result<*mut ffi::tidesdb_column_family_t> { + if cf.db.db == self.db { + Ok(cf.cf) + } else { + Err(Error::from_code( + ffi::TDB_ERR_INVALID_ARGS, + "column family belongs to a different database", + )) + } + } + + /// Writes a key-value pair that never expires. /// - /// # Arguments + /// An empty value is allowed and is distinct from an absent key: reads return it as an + /// empty `Vec`. + pub fn put(&self, cf: &ColumnFamily<'_>, key: &[u8], value: &[u8]) -> Result<()> { + self.put_raw(cf, key, value, 0) + } + + /// Writes a key-value pair that expires after `ttl`. /// - /// * `cf` - The column family - /// * `key` - The key - /// * `value` - The value - /// * `ttl` - Unix timestamp (seconds since epoch) for expiration, or -1 for no expiration - pub fn put(&self, cf: &ColumnFamily, key: &[u8], value: &[u8], ttl: i64) -> Result<()> { - let key_ptr = if key.is_empty() { - ptr::null() - } else { - key.as_ptr() - }; - let value_ptr = if value.is_empty() { - ptr::null() - } else { - value.as_ptr() - }; + /// The lifetime is rounded up to whole seconds and converted to an absolute deadline + /// when the write is buffered. A zero `ttl` is rejected. + pub fn put_with_ttl( + &self, + cf: &ColumnFamily<'_>, + key: &[u8], + value: &[u8], + ttl: Duration, + ) -> Result<()> { + if ttl.is_zero() { + return Err(Error::from_code( + ffi::TDB_ERR_INVALID_ARGS, + "ttl must be greater than zero", + )); + } + let secs = ttl + .as_secs() + .saturating_add(u64::from(ttl.subsec_nanos() > 0)); + let secs = libc::time_t::try_from(secs).unwrap_or(libc::time_t::MAX); + self.put_raw(cf, key, value, secs) + } + fn put_raw( + &self, + cf: &ColumnFamily<'_>, + key: &[u8], + value: &[u8], + ttl: libc::time_t, + ) -> Result<()> { let result = unsafe { ffi::tidesdb_txn_put( - self.txn, - cf.cf, - key_ptr, + self.txn(), + self.cf_ptr(cf)?, + key_ptr(key), key.len(), - value_ptr, + value.as_ptr(), value.len(), - ttl as libc::time_t, + ttl, ) }; check_result(result, "failed to put key-value pair") } - /// Retrieves a value from the transaction. - /// - /// # Arguments + /// Reads a key, recording the read for conflict detection. /// - /// * `cf` - The column family - /// * `key` - The key - /// - /// # Returns - /// - /// The value or an error if not found. - pub fn get(&self, cf: &ColumnFamily, key: &[u8]) -> Result> { - let key_ptr = if key.is_empty() { - ptr::null() - } else { - key.as_ptr() - }; + /// Returns an error with [`ErrorCode::NotFound`](crate::ErrorCode::NotFound) when the + /// key is absent, deleted or expired. + pub fn get(&self, cf: &ColumnFamily<'_>, key: &[u8]) -> Result> { + self.get_with(ffi::tidesdb_txn_get, cf, key, "failed to get value") + } + + /// Reads a key without recording it for conflict detection, for existence probes such + /// as uniqueness checks that should not add to the transaction's read set. + pub fn get_notrack(&self, cf: &ColumnFamily<'_>, key: &[u8]) -> Result> { + self.get_with(ffi::tidesdb_txn_get_notrack, cf, key, "failed to get value") + } + fn get_with( + &self, + f: unsafe extern "C" fn( + *mut ffi::tidesdb_txn_t, + *mut ffi::tidesdb_column_family_t, + *const u8, + usize, + *mut *mut u8, + *mut usize, + ) -> libc::c_int, + cf: &ColumnFamily<'_>, + key: &[u8], + context: &'static str, + ) -> Result> { let mut value_ptr: *mut u8 = ptr::null_mut(); let mut value_len: usize = 0; - let result = unsafe { - ffi::tidesdb_txn_get( - self.txn, - cf.cf, - key_ptr, + f( + self.txn(), + self.cf_ptr(cf)?, + key_ptr(key), key.len(), &mut value_ptr, &mut value_len, ) }; - check_result(result, "failed to get value")?; - - if value_ptr.is_null() { - return Err(Error::NullPointer("value")); - } + check_result(result, context)?; + Ok(unsafe { take_c_buffer(value_ptr, value_len) }) + } - let value = unsafe { - let slice = std::slice::from_raw_parts(value_ptr, value_len); - let vec = slice.to_vec(); - ffi::tidesdb_free(value_ptr as *mut std::ffi::c_void); - vec + /// Reads a key's expiry, as an absolute Unix timestamp in seconds. `Ok(None)` means the + /// key exists and never expires. + pub fn get_expiry(&self, cf: &ColumnFamily<'_>, key: &[u8]) -> Result> { + let mut ttl: libc::time_t = -1; + let result = unsafe { + ffi::tidesdb_txn_get_ttl( + self.txn(), + self.cf_ptr(cf)?, + key_ptr(key), + key.len(), + &mut ttl, + ) }; - - Ok(value) + check_result(result, "failed to get ttl")?; + Ok(ttl_from_c(ttl)) } - /// Removes a key-value pair from the transaction. - /// - /// # Arguments - /// - /// * `cf` - The column family - /// * `key` - The key - pub fn delete(&self, cf: &ColumnFamily, key: &[u8]) -> Result<()> { - let key_ptr = if key.is_empty() { - ptr::null() - } else { - key.as_ptr() + /// Whether a key exists, without recording the read for conflict detection. + pub fn contains(&self, cf: &ColumnFamily<'_>, key: &[u8]) -> Result { + let result = unsafe { + ffi::tidesdb_txn_contains(self.txn(), self.cf_ptr(cf)?, key_ptr(key), key.len()) }; + match result { + ffi::TDB_SUCCESS => Ok(true), + ffi::TDB_ERR_NOT_FOUND => Ok(false), + code => Err(Error::from_code(code, "failed to check key")), + } + } - let result = unsafe { ffi::tidesdb_txn_delete(self.txn, cf.cf, key_ptr, key.len()) }; + /// Deletes a key. + pub fn delete(&self, cf: &ColumnFamily<'_>, key: &[u8]) -> Result<()> { + let result = unsafe { + ffi::tidesdb_txn_delete(self.txn(), self.cf_ptr(cf)?, key_ptr(key), key.len()) + }; check_result(result, "failed to delete key") } /// Writes a single-delete tombstone for the key. /// - /// `single_delete` has the same read semantics as [`delete`](Self::delete), but - /// carries a caller-provided promise that lets compaction drop the put and the - /// tombstone together as soon as both appear in the same merge input, rather than - /// carrying the tombstone forward until the largest active level. - /// - /// # Contract - /// - /// Between any two single-deletes on the same key (and between the start of the - /// key's history and its first single-delete), the key must have been put **at - /// most once**. The engine does not verify this at runtime; violating the contract - /// can leave older puts visible after the single-delete and is a bug in the caller. - /// - /// Use this for workloads where each key is inserted exactly once and then deleted - /// exactly once (insert-benchmark patterns, secondary-index entries on immutable - /// columns, log-style tables with scheduled purges). It is **not** safe for tables - /// that issue repeated updates to the same key. When in doubt, use + /// Reads behave as for [`delete`](Self::delete), but the caller promises the key was + /// put **at most once** since its previous single-delete (or since it was created), so + /// compaction can drop the put and the tombstone together. The engine does not verify + /// this; breaking the contract can leave older puts visible. When in doubt, use /// [`delete`](Self::delete). + pub fn single_delete(&self, cf: &ColumnFamily<'_>, key: &[u8]) -> Result<()> { + let result = unsafe { + ffi::tidesdb_txn_single_delete(self.txn(), self.cf_ptr(cf)?, key_ptr(key), key.len()) + }; + check_result(result, "failed to single-delete key") + } + + /// Deletes every key in `[lo, hi)`, or from `lo` to the end of the family when `hi` is + /// `None`. /// - /// Requires tidesdb >= 9.1.0 (the `v9_1_0` Cargo feature). - /// - /// # Arguments - /// - /// * `cf` - The column family - /// * `key` - The key - #[cfg(tidesdb_has_txn_single_delete)] - pub fn single_delete(&self, cf: &ColumnFamily, key: &[u8]) -> Result<()> { - let key_ptr = if key.is_empty() { - ptr::null() - } else { - key.as_ptr() + /// It costs one entry however many keys it covers, and it covers keys written before + /// it, not merely the keys present when it is called; a newer write inside the range + /// survives it. `lo` must be non-empty, `hi` (when given) greater than `lo`, and each + /// bound at most 256 bytes; otherwise [`ErrorCode::InvalidArgs`](crate::ErrorCode::InvalidArgs). + pub fn delete_range(&self, cf: &ColumnFamily<'_>, lo: &[u8], hi: Option<&[u8]>) -> Result<()> { + // the C library reads an empty upper bound as "no upper bound", which would turn an + // empty interval into a delete of everything from `lo` on + if hi.is_some_and(|h| h.is_empty()) { + return Err(Error::from_code( + ffi::TDB_ERR_INVALID_ARGS, + "range delete upper bound must be non-empty; use None for no upper bound", + )); + } + let (hi_ptr, hi_len) = hi.map_or((ptr::null(), 0), |h| (h.as_ptr(), h.len())); + let result = unsafe { + ffi::tidesdb_txn_delete_range( + self.txn(), + self.cf_ptr(cf)?, + key_ptr(lo), + lo.len(), + hi_ptr, + hi_len, + ) }; + check_result(result, "failed to delete range") + } - let result = unsafe { ffi::tidesdb_txn_single_delete(self.txn, cf.cf, key_ptr, key.len()) }; - check_result(result, "failed to single-delete key") + /// Deletes every key that begins with `prefix` (non-empty, at most 256 bytes). + /// + /// Same semantics as [`delete_range`](Self::delete_range). + pub fn delete_prefix(&self, cf: &ColumnFamily<'_>, prefix: &[u8]) -> Result<()> { + let result = unsafe { + ffi::tidesdb_txn_delete_prefix( + self.txn(), + self.cf_ptr(cf)?, + key_ptr(prefix), + prefix.len(), + ) + }; + check_result(result, "failed to delete prefix") } /// Commits the transaction. /// - /// After committing, the transaction cannot be used for further operations - /// unless `reset` is called. + /// Returns [`ErrorCode::Conflict`](crate::ErrorCode::Conflict) when another transaction + /// committed a conflicting write first. pub fn commit(&mut self) -> Result<()> { - let result = unsafe { ffi::tidesdb_txn_commit(self.txn) }; - self.committed = true; + let result = unsafe { ffi::tidesdb_txn_commit(self.txn()) }; check_result(result, "failed to commit transaction") } - /// Rolls back the transaction. - /// - /// After rolling back, the transaction cannot be used for further operations - /// unless `reset` is called. + /// Discards the transaction's buffered writes. pub fn rollback(&mut self) -> Result<()> { - let result = unsafe { ffi::tidesdb_txn_rollback(self.txn) }; - self.committed = true; // Mark as done to prevent double-free + let result = unsafe { ffi::tidesdb_txn_rollback(self.txn()) }; check_result(result, "failed to rollback transaction") } - /// Resets a committed or aborted transaction for reuse with a new isolation level. - /// This avoids the overhead of freeing and reallocating transaction resources. - /// - /// # Arguments - /// - /// * `isolation` - The new isolation level for the reset transaction - /// - /// # Errors - /// - /// Returns an error if the transaction is still active (not committed/aborted) - /// or if the isolation level is invalid. + /// Discards the transaction's state, committed or not, and begins it again at a new + /// isolation level and a fresh snapshot. The database's default timeout is restored and + /// any pending abort request is cleared. pub fn reset(&mut self, isolation: IsolationLevel) -> Result<()> { - let result = unsafe { ffi::tidesdb_txn_reset(self.txn, isolation as i32) }; - check_result(result, "failed to reset transaction")?; - self.committed = false; - Ok(()) + let result = unsafe { ffi::tidesdb_txn_reset(self.txn(), isolation as i32) }; + check_result(result, "failed to reset transaction") } - /// Creates a savepoint within the transaction. + /// Bounds how long this transaction may stay active, overriding the database's + /// `txn_timeout_seconds`. Measured from this call; `None` clears the timeout. /// - /// # Arguments + /// The transaction is not aborted in the background: its next operation notices and + /// fails with [`ErrorCode::TxnExpired`](crate::ErrorCode::TxnExpired). Deadlines are + /// checked against the engine's one-second clock, so expiry can land up to about two + /// seconds after the nominal deadline. + pub fn set_timeout(&mut self, timeout: Option) -> Result<()> { + let seconds = timeout.map_or(0, |t| { + let secs = t.as_secs().saturating_add(u64::from(t.subsec_nanos() > 0)); + i64::try_from(secs).unwrap_or(i64::MAX).max(1) + }); + let result = unsafe { ffi::tidesdb_txn_set_timeout(self.txn(), seconds) }; + check_result(result, "failed to set transaction timeout") + } + + /// The sequence ceiling this transaction's reads see: `u64::MAX` for read uncommitted, + /// the current sequence for read committed, and the sequence frozen at begin for + /// repeatable read and stronger. + pub fn read_snapshot(&self) -> u64 { + unsafe { ffi::tidesdb_txn_read_snapshot(self.txn()) } + } + + /// The transaction's lifecycle state. + pub fn state(&self) -> Result { + let mut state: libc::c_int = 0; + let result = unsafe { ffi::tidesdb_txn_state(self.txn(), &mut state) }; + check_result(result, "failed to get transaction state")?; + Ok(match state { + ffi::TDB_TXN_STATE_ACTIVE => TxnState::Active, + ffi::TDB_TXN_STATE_PREPARED => TxnState::Prepared, + ffi::TDB_TXN_STATE_COMMITTED => TxnState::Committed, + _ => TxnState::Aborted, + }) + } + + /// Returns a handle that can abort this transaction from another thread. + pub fn abort_handle(&self) -> AbortHandle<'db> { + AbortHandle { + handle: Arc::clone(&self.handle), + _marker: PhantomData, + } + } + + // ===== two-phase commit ===== + + /// Two-phase commit, phase one: runs the commit's conflict checks and durably logs the + /// write batch under `xid`, leaving the writes invisible until + /// [`commit_prepared`](Self::commit_prepared) or [`rollback_prepared`](Self::rollback_prepared). /// - /// * `name` - The savepoint name + /// Dropping a prepared transaction without resolving it leaves it in doubt on disk; after + /// the next open it is listed by [`TidesDB::recover_prepared`]. + pub fn prepare(&mut self, xid: &[u8]) -> Result<()> { + let result = unsafe { ffi::tidesdb_txn_prepare(self.txn(), xid.as_ptr(), xid.len()) }; + check_result(result, "failed to prepare transaction") + } + + /// Two-phase commit, phase two: commits a prepared transaction. A transient I/O failure + /// leaves it prepared so the call can be retried. + pub fn commit_prepared(&mut self) -> Result<()> { + let result = unsafe { ffi::tidesdb_txn_commit_prepared(self.txn()) }; + check_result(result, "failed to commit prepared transaction") + } + + /// Two-phase commit, phase two: rolls back a prepared transaction. + pub fn rollback_prepared(&mut self) -> Result<()> { + let result = unsafe { ffi::tidesdb_txn_rollback_prepared(self.txn()) }; + check_result(result, "failed to rollback prepared transaction") + } + + // ===== savepoints ===== + + /// Marks a named savepoint. pub fn savepoint(&self, name: &str) -> Result<()> { let c_name = CString::new(name)?; - let result = unsafe { ffi::tidesdb_txn_savepoint(self.txn, c_name.as_ptr()) }; + let result = unsafe { ffi::tidesdb_txn_savepoint(self.txn(), c_name.as_ptr()) }; check_result(result, "failed to create savepoint") } - /// Rolls back the transaction to a savepoint. - /// - /// # Arguments - /// - /// * `name` - The savepoint name + /// Discards writes buffered since the named savepoint. pub fn rollback_to_savepoint(&self, name: &str) -> Result<()> { let c_name = CString::new(name)?; - let result = unsafe { ffi::tidesdb_txn_rollback_to_savepoint(self.txn, c_name.as_ptr()) }; + let result = unsafe { ffi::tidesdb_txn_rollback_to_savepoint(self.txn(), c_name.as_ptr()) }; check_result(result, "failed to rollback to savepoint") } /// Releases a savepoint without rolling back. - /// - /// # Arguments - /// - /// * `name` - The savepoint name pub fn release_savepoint(&self, name: &str) -> Result<()> { let c_name = CString::new(name)?; - let result = unsafe { ffi::tidesdb_txn_release_savepoint(self.txn, c_name.as_ptr()) }; + let result = unsafe { ffi::tidesdb_txn_release_savepoint(self.txn(), c_name.as_ptr()) }; check_result(result, "failed to release savepoint") } - /// Creates a new iterator for a column family within this transaction. - /// - /// # Arguments - /// - /// * `cf` - The column family - /// - /// # Returns + // ===== iterators ===== + + /// Creates an iterator over a whole column family at this transaction's snapshot. /// - /// A new iterator. - pub fn new_iterator(&self, cf: &ColumnFamily) -> Result { + /// The iterator borrows the transaction, so it must be dropped before commit. At + /// repeatable read and serializable that is also what records the scanned range for + /// conflict detection. + pub fn new_iterator(&self, cf: &ColumnFamily<'_>) -> Result> { let mut iter: *mut ffi::tidesdb_iter_t = ptr::null_mut(); - - let result = unsafe { ffi::tidesdb_iter_new(self.txn, cf.cf, &mut iter) }; + let result = unsafe { ffi::tidesdb_iter_new(self.txn(), self.cf_ptr(cf)?, &mut iter) }; check_result(result, "failed to create iterator")?; - if iter.is_null() { return Err(Error::NullPointer("iterator handle")); } + Ok(unsafe { Iterator::from_raw(iter) }) + } - Ok(Iterator::new(iter)) + /// Creates an iterator that only opens the sstables overlapping `[lower, upper]`. + /// + /// Much cheaper than [`new_iterator`](Self::new_iterator) for a narrow scan of a large + /// family. The range is a promise, not a fence: results are only defined inside it, and + /// seeking or stepping outside it may report a key that exists as absent. + pub fn new_range_iterator( + &self, + cf: &ColumnFamily<'_>, + lower: &[u8], + upper: &[u8], + ) -> Result> { + let mut iter: *mut ffi::tidesdb_iter_t = ptr::null_mut(); + let result = unsafe { + ffi::tidesdb_iter_new_range( + self.txn(), + self.cf_ptr(cf)?, + key_ptr(lower), + lower.len(), + key_ptr(upper), + upper.len(), + &mut iter, + ) + }; + check_result(result, "failed to create range iterator")?; + if iter.is_null() { + return Err(Error::NullPointer("iterator handle")); + } + Ok(unsafe { Iterator::from_raw(iter) }) } } -impl Drop for Transaction { +impl Drop for Transaction<'_> { fn drop(&mut self) { - if !self.txn.is_null() { - unsafe { - ffi::tidesdb_txn_free(self.txn); - } - self.txn = ptr::null_mut(); + // If an AbortHandle still holds the C transaction, it is freed when that handle + // drops; roll back now so it stops holding its snapshot in the meantime. + if Arc::strong_count(&self.handle) > 1 { + unsafe { ffi::tidesdb_txn_rollback(self.txn()) }; } } } + +/// Aborts a [`Transaction`] from another thread. +/// +/// Obtained from [`Transaction::abort_handle`]. The handle is `Send + Sync` and may outlive +/// the transaction (but not the database); aborting a finished transaction does nothing. +#[derive(Clone)] +pub struct AbortHandle<'db> { + handle: Arc, + _marker: PhantomData<&'db TidesDB>, +} + +impl AbortHandle<'_> { + /// Asks the transaction to abort. + /// + /// This only sets a flag: the owning thread's next operation on the transaction aborts + /// it and returns [`ErrorCode::TxnAborted`](crate::ErrorCode::TxnAborted), which is + /// distinct from the engine's own [`ErrorCode::Conflict`](crate::ErrorCode::Conflict). + /// A transaction that has already prepared is not affected. + pub fn request_abort(&self) { + unsafe { ffi::tidesdb_txn_request_abort(self.handle.txn) }; + } +} diff --git a/tidesdb b/tidesdb new file mode 160000 index 0000000..abd8900 --- /dev/null +++ b/tidesdb @@ -0,0 +1 @@ +Subproject commit abd8900c9c5fd8c673ca5481209ee99689bba779 From 56fcb0c2af113d197b6ecb5b67006334bd082f87 Mon Sep 17 00:00:00 2001 From: Alex Gaetano Padula Date: Wed, 14 Oct 2026 14:36:58 -0400 Subject: [PATCH 2/2] detect c time_t width for 32-bit targets, rerun build on tidesdb.pc changes --- Cargo.toml | 1 + build.rs | 56 ++++++++++++++++++++++++++++++++++++++++++++++ doc/rust.md | 2 ++ src/db.rs | 2 +- src/ffi.rs | 10 ++++++++- src/iterator.rs | 2 +- src/transaction.rs | 6 ++--- 7 files changed, 73 insertions(+), 6 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 2ec5d24..264bae7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,6 +37,7 @@ thiserror = "1.0" [build-dependencies] pkg-config = "0.3" cmake = "0.1" +cc = "1" [dev-dependencies] tempfile = "3.10" diff --git a/build.rs b/build.rs index 688d2de..b4fde0d 100644 --- a/build.rs +++ b/build.rs @@ -226,6 +226,19 @@ fn link_allocator() { /// Use a TidesDB found through pkg-config if it belongs to the bound major and is at least the /// bundled release. Returns the version linked, or `None` to build the bundled source. +/// The `.pc` file pkg-config resolves `name` to. +fn pc_file(name: &str) -> Option { + let pkg_config = std::env::var("PKG_CONFIG").unwrap_or_else(|_| "pkg-config".into()); + let out = std::process::Command::new(pkg_config) + .args(["--variable=pcfiledir", name]) + .output() + .ok() + .filter(|o| o.status.success())?; + let dir = String::from_utf8_lossy(&out.stdout).trim().to_string(); + let pc = PathBuf::from(dir).join(format!("{name}.pc")); + pc.exists().then_some(pc) +} + fn system_tidesdb(major: u64) -> Option { if std::env::var_os("TIDESDB_NO_PKG_CONFIG").is_some() { return None; @@ -236,6 +249,10 @@ fn system_tidesdb(major: u64) -> Option { .env_metadata(true) .probe("tidesdb") .ok()?; + // rebuild when the installed TidesDB changes (an upgrade rewrites its .pc file) + if let Some(pc) = pc_file("tidesdb") { + println!("cargo:rerun-if-changed={}", pc.display()); + } let crate_version = env!("CARGO_PKG_VERSION"); let crate_major = major - TIDESDB_MAJOR_OFFSET; let usable = match parse_version(&found.version) { @@ -277,6 +294,40 @@ fn system_tidesdb(major: u64) -> Option { Some(lib.version) } +/// The width of the C `time_t` on the target, in bits, asked of the target's C compiler. +/// +/// It is not fixed per target: 32-bit glibc targets have a 32-bit `time_t` by default but a +/// 64-bit one where the toolchain defines `_TIME_BITS=64` (Debian and Ubuntu armhf since the +/// 64-bit time transition), while Rust's `libc::time_t` stays 32-bit. `time_t` is a field of +/// `tidesdb_commit_op_t` and a TTL parameter, so the bindings must match the compiler that +/// built TidesDB. Each candidate is checked with a compile-time assertion, so this also works +/// when cross-compiling. +fn c_time_t_bits() -> u32 { + let out = PathBuf::from(std::env::var("OUT_DIR").unwrap()); + for bits in [64u32, 32] { + let probe = out.join(format!("time_t_{bits}.c")); + std::fs::write( + &probe, + format!( + "#include \n\ + typedef char tidesdb_time_t_probe[sizeof(time_t) == {} ? 1 : -1];\n", + bits / 8 + ), + ) + .unwrap(); + let compiled = cc::Build::new() + .file(&probe) + .cargo_metadata(false) + .cargo_warnings(false) + .warnings(false) + .try_compile_intermediates(); + if compiled.is_ok() { + return bits; + } + } + panic!("could not determine the size of the C time_t for this target"); +} + /// The platform the crate is being built for, from Cargo's target cfg. A build script's own /// `#[cfg(...)]` describes the machine running the build, which differs when cross-compiling. struct Target { @@ -385,10 +436,15 @@ fn main() { println!("cargo:rerun-if-env-changed=TIDESDB_NO_PKG_CONFIG"); println!("cargo:rerun-if-env-changed=MINGW_PREFIX"); println!("cargo:rustc-check-cfg=cfg(tidesdb_system)"); + println!("cargo:rustc-check-cfg=cfg(tidesdb_time_t_32)"); let major = tidesdb_major(); let target = Target::from_env(); + if c_time_t_bits() == 32 { + println!("cargo:rustc-cfg=tidesdb_time_t_32"); + } + if let Some(version) = system_tidesdb(major) { println!("cargo:rustc-cfg=tidesdb_system"); println!("cargo:rustc-env=TIDESDB_LINKED_VERSION={version}"); diff --git a/doc/rust.md b/doc/rust.md index b0821c4..6da9998 100644 --- a/doc/rust.md +++ b/doc/rust.md @@ -213,6 +213,8 @@ The build script links the system libraries the static TidesDB needs for the **t A 32-bit target's toolchain must provide `libatomic` (GCC cross toolchains do). +The width of the C `time_t` differs between 32-bit toolchains: Debian and Ubuntu armhf (and others that define `_TIME_BITS=64`) use 64 bits, older ones 32. TidesDB passes `time_t` in commit-hook ops and TTL calls, so the build script asks the target C compiler which width it uses and the bindings follow it. Use the same toolchain for the whole build. + Package-manager library directories (Homebrew, `/usr/local/lib`, MSYS2, pkgsrc) are added to the link search path only for native builds. When cross-compiling, the compression and allocator libraries must be built for the target. Make them visible through pkg-config, which the build uses only when cross-compiling is allowed (`PKG_CONFIG_ALLOW_CROSS=1`, with `PKG_CONFIG_SYSROOT_DIR` and `PKG_CONFIG_PATH` pointing into the target sysroot), or with `RUSTFLAGS="-L /path/to/target/libs"`. The same applies to a system TidesDB: when cross-compiling, pkg-config only finds one if it is configured for the target, and otherwise the bundled source is built. The CI cross-builds and tests `armv7-unknown-linux-gnueabihf` and `aarch64-unknown-linux-gnu` under QEMU, and cross-builds `x86_64-pc-windows-gnu` from Linux. diff --git a/src/db.rs b/src/db.rs index dabf5a9..8616357 100644 --- a/src/db.rs +++ b/src/db.rs @@ -118,7 +118,7 @@ pub(crate) unsafe fn take_c_buffer(ptr: *mut u8, len: usize) -> Vec { /// Convert a C absolute expiry (`-1` = never) to an `Option`. // time_t is not i64 on every target, so the conversion is not always a no-op #[allow(clippy::useless_conversion)] -pub(crate) fn ttl_from_c(ttl: libc::time_t) -> Option { +pub(crate) fn ttl_from_c(ttl: ffi::time_t) -> Option { if ttl < 0 { None } else { Some(i64::from(ttl)) } } diff --git a/src/ffi.rs b/src/ffi.rs index 31f8ef4..be38738 100644 --- a/src/ffi.rs +++ b/src/ffi.rs @@ -15,7 +15,15 @@ #![allow(dead_code)] #![allow(non_camel_case_types)] -use libc::{c_char, c_double, c_float, c_int, c_long, c_void, size_t, time_t}; +use libc::{c_char, c_double, c_float, c_int, c_long, c_void, size_t}; + +/// The C `time_t` of the compiler that built TidesDB. `libc::time_t` is not used because it +/// is 32-bit on 32-bit glibc targets even where the C toolchain defaults to a 64-bit +/// `time_t`; build.rs asks the target compiler and sets `tidesdb_time_t_32` when it is 32-bit. +#[cfg(not(tidesdb_time_t_32))] +pub type time_t = i64; +#[cfg(tidesdb_time_t_32)] +pub type time_t = i32; /// The TidesDB release these bindings were written against. pub const TIDESDB_VERSION: &str = "10.2.1"; diff --git a/src/iterator.rs b/src/iterator.rs index 52cd9de..f58f1e6 100644 --- a/src/iterator.rs +++ b/src/iterator.rs @@ -151,7 +151,7 @@ impl Iterator<'_> { /// The current entry's expiry as an absolute Unix timestamp in seconds, `None` if it /// never expires. pub fn expires_at(&self) -> Result> { - let mut ttl: libc::time_t = -1; + let mut ttl: ffi::time_t = -1; let result = unsafe { ffi::tidesdb_iter_ttl(self.iter, &mut ttl) }; check_result(result, "failed to get ttl")?; Ok(ttl_from_c(ttl)) diff --git a/src/transaction.rs b/src/transaction.rs index 076f1fc..97e0258 100644 --- a/src/transaction.rs +++ b/src/transaction.rs @@ -144,7 +144,7 @@ impl<'db> Transaction<'db> { let secs = ttl .as_secs() .saturating_add(u64::from(ttl.subsec_nanos() > 0)); - let secs = libc::time_t::try_from(secs).unwrap_or(libc::time_t::MAX); + let secs = ffi::time_t::try_from(secs).unwrap_or(ffi::time_t::MAX); self.put_raw(cf, key, value, secs) } @@ -153,7 +153,7 @@ impl<'db> Transaction<'db> { cf: &ColumnFamily<'_>, key: &[u8], value: &[u8], - ttl: libc::time_t, + ttl: ffi::time_t, ) -> Result<()> { let result = unsafe { ffi::tidesdb_txn_put( @@ -216,7 +216,7 @@ impl<'db> Transaction<'db> { /// Reads a key's expiry, as an absolute Unix timestamp in seconds. `Ok(None)` means the /// key exists and never expires. pub fn get_expiry(&self, cf: &ColumnFamily<'_>, key: &[u8]) -> Result> { - let mut ttl: libc::time_t = -1; + let mut ttl: ffi::time_t = -1; let result = unsafe { ffi::tidesdb_txn_get_ttl( self.txn(),